{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# G-Research Crypto Train Data EDA\nhttps://www.kaggle.com/stpeteishii/g-research-crypto-train-data-eda","metadata":{}},{"cell_type":"code","source":"import gc\nfrom tqdm import tqdm\nimport numpy as np\nimport pandas as pd\nfrom pylab import rcParams\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import probplot\nimport datetime as dt\nfrom datetime import date\nfrom datetime import timedelta\n\n%matplotlib inline\nimport os\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.462466,"end_time":"2021-12-27T13:58:32.372986","exception":false,"start_time":"2021-12-27T13:58:30.91052","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-31T02:22:11.227404Z","iopub.execute_input":"2021-12-31T02:22:11.227697Z","iopub.status.idle":"2021-12-31T02:22:11.236271Z","shell.execute_reply.started":"2021-12-31T02:22:11.227668Z","shell.execute_reply":"2021-12-31T02:22:11.235566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Data Processing","metadata":{}},{"cell_type":"code","source":"data0 = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\ndata0[-5:]","metadata":{"papermill":{"duration":62.894621,"end_time":"2021-12-27T13:59:35.280079","exception":false,"start_time":"2021-12-27T13:58:32.385458","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-31T02:22:11.237714Z","iopub.execute_input":"2021-12-31T02:22:11.238279Z","iopub.status.idle":"2021-12-31T02:22:49.675119Z","shell.execute_reply.started":"2021-12-31T02:22:11.238249Z","shell.execute_reply":"2021-12-31T02:22:49.674567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0['ds']=pd.to_datetime(data0['timestamp']*10**9,format='%Y-%m-%d %H:%M:%S')","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:49.676125Z","iopub.execute_input":"2021-12-31T02:22:49.676444Z","iopub.status.idle":"2021-12-31T02:22:51.799827Z","shell.execute_reply.started":"2021-12-31T02:22:49.676415Z","shell.execute_reply":"2021-12-31T02:22:51.799041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0a=data0[['ds','Asset_ID','Target']]\ndata0a","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:51.802168Z","iopub.execute_input":"2021-12-31T02:22:51.802501Z","iopub.status.idle":"2021-12-31T02:22:52.03873Z","shell.execute_reply.started":"2021-12-31T02:22:51.80246Z","shell.execute_reply":"2021-12-31T02:22:52.037863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data0","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:52.04007Z","iopub.execute_input":"2021-12-31T02:22:52.040465Z","iopub.status.idle":"2021-12-31T02:22:52.047108Z","shell.execute_reply.started":"2021-12-31T02:22:52.040413Z","shell.execute_reply":"2021-12-31T02:22:52.046423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0a.info()","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:52.048648Z","iopub.execute_input":"2021-12-31T02:22:52.048849Z","iopub.status.idle":"2021-12-31T02:22:52.063646Z","shell.execute_reply.started":"2021-12-31T02:22:52.048824Z","shell.execute_reply":"2021-12-31T02:22:52.062656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Period of the train data","metadata":{}},{"cell_type":"code","source":"import datetime as dt\nfrom datetime import datetime as dt\nfrom datetime import date\nfrom datetime import timedelta\n\nstart0 = dt.strptime('2018-01-01 00:01:00','%Y-%m-%d %H:%M:%S')\nend0   = dt.strptime('2021-09-21 00:00:00','%Y-%m-%d %H:%M:%S')\n\nprint(end0-start0)\nseconds0=(end0-start0).days*24*60*60+(end0-start0).seconds\nprint(seconds0)\n\ntime=[]\nfor i in list(range(0,seconds0+60,60)):\n    time+=[start0+timedelta(seconds=i)]\ndates0=pd.to_datetime(time,format='%Y-%m-%d %H:%M:%S')\nprint(dates0)\n\n# train period\ndates0_df=pd.DataFrame(dates0)\ndates0_df.columns=['ds']\ndates0_df","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:52.064641Z","iopub.execute_input":"2021-12-31T02:22:52.064866Z","iopub.status.idle":"2021-12-31T02:22:55.063843Z","shell.execute_reply.started":"2021-12-31T02:22:52.064838Z","shell.execute_reply":"2021-12-31T02:22:55.063174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0a0=data0a[['ds','Target']][data0a['Asset_ID']==0]\ndata0a0.columns=['ds','Target:0']\n\ndata0a1=data0a[['ds','Target']][data0a['Asset_ID']==1]\ndata0a1.columns=['ds','Target:1']\n\ndata0a2=data0a[['ds','Target']][data0a['Asset_ID']==2]\ndata0a2.columns=['ds','Target:2']\n\ndata0a3=data0a[['ds','Target']][data0a['Asset_ID']==3]\ndata0a3.columns=['ds','Target:3']\n\ndata0a4=data0a[['ds','Target']][data0a['Asset_ID']==4]\ndata0a4.columns=['ds','Target:4']\n\ndata0a5=data0a[['ds','Target']][data0a['Asset_ID']==5]\ndata0a5.columns=['ds','Target:5']\n\ndata0a6=data0a[['ds','Target']][data0a['Asset_ID']==6]\ndata0a6.columns=['ds','Target:6']\n\ndata0a7=data0a[['ds','Target']][data0a['Asset_ID']==7]\ndata0a7.columns=['ds','Target:7']\n\ndata0a8=data0a[['ds','Target']][data0a['Asset_ID']==8]\ndata0a8.columns=['ds','Target:8']\n\ndata0a9=data0a[['ds','Target']][data0a['Asset_ID']==9]\ndata0a9.columns=['ds','Target:9']\n\ndata0a10=data0a[['ds','Target']][data0a['Asset_ID']==10]\ndata0a10.columns=['ds','Target:10']\n\ndata0a11=data0a[['ds','Target']][data0a['Asset_ID']==11]\ndata0a11.columns=['ds','Target:11']\n\ndata0a12=data0a[['ds','Target']][data0a['Asset_ID']==12]\ndata0a12.columns=['ds','Target:12']\n\ndata0a13=data0a[['ds','Target']][data0a['Asset_ID']==13]\ndata0a13.columns=['ds','Target:13']","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:55.064745Z","iopub.execute_input":"2021-12-31T02:22:55.065457Z","iopub.status.idle":"2021-12-31T02:22:58.856245Z","shell.execute_reply.started":"2021-12-31T02:22:55.065421Z","shell.execute_reply":"2021-12-31T02:22:58.855554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0b=dates0_df","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:58.858603Z","iopub.execute_input":"2021-12-31T02:22:58.858812Z","iopub.status.idle":"2021-12-31T02:22:58.868872Z","shell.execute_reply.started":"2021-12-31T02:22:58.858786Z","shell.execute_reply":"2021-12-31T02:22:58.867952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0a0","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:58.888109Z","iopub.execute_input":"2021-12-31T02:22:58.888456Z","iopub.status.idle":"2021-12-31T02:22:58.907052Z","shell.execute_reply.started":"2021-12-31T02:22:58.888423Z","shell.execute_reply":"2021-12-31T02:22:58.906042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0b=data0b.merge(data0a0,how='left', on='ds')\ndata0b=data0b.merge(data0a1,how='left',on='ds')\ndata0b=data0b.merge(data0a2,how='left',on='ds')\ndata0b=data0b.merge(data0a3,how='left',on='ds')\ndata0b=data0b.merge(data0a4,how='left',on='ds')\ndata0b=data0b.merge(data0a5,how='left',on='ds')\ndata0b=data0b.merge(data0a6,how='left',on='ds')\ndata0b=data0b.merge(data0a7,how='left',on='ds')\ndata0b=data0b.merge(data0a8,how='left',on='ds')\ndata0b=data0b.merge(data0a9,how='left',on='ds')\ndata0b=data0b.merge(data0a10,how='left',on='ds')\ndata0b=data0b.merge(data0a11,how='left',on='ds')\ndata0b=data0b.merge(data0a11,how='left',on='ds')\ndata0b=data0b.merge(data0a13,how='left',on='ds')","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:22:58.908756Z","iopub.execute_input":"2021-12-31T02:22:58.909189Z","iopub.status.idle":"2021-12-31T02:23:12.61463Z","shell.execute_reply.started":"2021-12-31T02:22:58.909121Z","shell.execute_reply":"2021-12-31T02:23:12.613204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data0b","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:12.616621Z","iopub.execute_input":"2021-12-31T02:23:12.616936Z","iopub.status.idle":"2021-12-31T02:23:12.649905Z","shell.execute_reply.started":"2021-12-31T02:23:12.616899Z","shell.execute_reply":"2021-12-31T02:23:12.649049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"item1=data0b.copy()\nitem1.columns = ['ds','y1','y2','y3','y4','y5','y6','y7','y8','y9','y10','y11','y12','y13','y14']\nitem1.y1 = item1.y1.astype('float')\nitem1.y2 = item1.y2.astype('float')\nitem1.y3 = item1.y3.astype('float')\nitem1.y4 = item1.y4.astype('float')\nitem1.y5 = item1.y5.astype('float')\nitem1.y6 = item1.y6.astype('float')\nitem1.y7 = item1.y7.astype('float')\nitem1.y8 = item1.y8.astype('float')\nitem1.y9 = item1.y9.astype('float')\nitem1.y10 = item1.y10.astype('float')\nitem1.y11 = item1.y11.astype('float')\nitem1.y12 = item1.y12.astype('float')\nitem1.y13 = item1.y13.astype('float')\nitem1.y14 = item1.y14.astype('float')\nitem1.ds = item1.ds.astype('datetime64')","metadata":{"papermill":{"duration":0.013419,"end_time":"2021-12-27T13:59:38.62241","exception":false,"start_time":"2021-12-27T13:59:38.608991","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-31T02:23:12.651499Z","iopub.execute_input":"2021-12-31T02:23:12.652198Z","iopub.status.idle":"2021-12-31T02:23:13.148847Z","shell.execute_reply.started":"2021-12-31T02:23:12.652155Z","shell.execute_reply":"2021-12-31T02:23:13.147993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data0a\ndel data0b","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:13.150612Z","iopub.execute_input":"2021-12-31T02:23:13.151189Z","iopub.status.idle":"2021-12-31T02:23:13.155096Z","shell.execute_reply.started":"2021-12-31T02:23:13.151148Z","shell.execute_reply":"2021-12-31T02:23:13.154347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y1)","metadata":{"papermill":{"duration":0.327863,"end_time":"2021-12-27T13:59:38.990748","exception":false,"start_time":"2021-12-27T13:59:38.662885","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-12-31T02:23:13.156401Z","iopub.execute_input":"2021-12-31T02:23:13.156627Z","iopub.status.idle":"2021-12-31T02:23:14.027844Z","shell.execute_reply.started":"2021-12-31T02:23:13.156601Z","shell.execute_reply":"2021-12-31T02:23:14.027024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y2)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:14.029392Z","iopub.execute_input":"2021-12-31T02:23:14.029687Z","iopub.status.idle":"2021-12-31T02:23:14.807136Z","shell.execute_reply.started":"2021-12-31T02:23:14.029646Z","shell.execute_reply":"2021-12-31T02:23:14.806509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y3)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:14.808216Z","iopub.execute_input":"2021-12-31T02:23:14.808477Z","iopub.status.idle":"2021-12-31T02:23:15.574151Z","shell.execute_reply.started":"2021-12-31T02:23:14.808447Z","shell.execute_reply":"2021-12-31T02:23:15.573449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y4)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:15.575349Z","iopub.execute_input":"2021-12-31T02:23:15.575566Z","iopub.status.idle":"2021-12-31T02:23:16.39467Z","shell.execute_reply.started":"2021-12-31T02:23:15.575539Z","shell.execute_reply":"2021-12-31T02:23:16.393737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y5)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:16.395872Z","iopub.execute_input":"2021-12-31T02:23:16.396109Z","iopub.status.idle":"2021-12-31T02:23:17.246896Z","shell.execute_reply.started":"2021-12-31T02:23:16.39608Z","shell.execute_reply":"2021-12-31T02:23:17.246102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y6)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:17.248251Z","iopub.execute_input":"2021-12-31T02:23:17.248573Z","iopub.status.idle":"2021-12-31T02:23:18.048885Z","shell.execute_reply.started":"2021-12-31T02:23:17.248527Z","shell.execute_reply":"2021-12-31T02:23:18.048093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y7)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:18.05012Z","iopub.execute_input":"2021-12-31T02:23:18.050343Z","iopub.status.idle":"2021-12-31T02:23:18.806215Z","shell.execute_reply.started":"2021-12-31T02:23:18.050315Z","shell.execute_reply":"2021-12-31T02:23:18.80535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y8)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:18.807251Z","iopub.execute_input":"2021-12-31T02:23:18.807466Z","iopub.status.idle":"2021-12-31T02:23:19.586335Z","shell.execute_reply.started":"2021-12-31T02:23:18.807441Z","shell.execute_reply":"2021-12-31T02:23:19.58554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y9)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:19.58968Z","iopub.execute_input":"2021-12-31T02:23:19.589895Z","iopub.status.idle":"2021-12-31T02:23:20.56523Z","shell.execute_reply.started":"2021-12-31T02:23:19.589868Z","shell.execute_reply":"2021-12-31T02:23:20.564448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y10)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:20.566353Z","iopub.execute_input":"2021-12-31T02:23:20.566587Z","iopub.status.idle":"2021-12-31T02:23:21.362748Z","shell.execute_reply.started":"2021-12-31T02:23:20.566559Z","shell.execute_reply":"2021-12-31T02:23:21.361844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y11)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:21.363951Z","iopub.execute_input":"2021-12-31T02:23:21.364185Z","iopub.status.idle":"2021-12-31T02:23:22.012667Z","shell.execute_reply.started":"2021-12-31T02:23:21.364156Z","shell.execute_reply":"2021-12-31T02:23:22.011744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y12)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:22.013796Z","iopub.execute_input":"2021-12-31T02:23:22.014023Z","iopub.status.idle":"2021-12-31T02:23:23.031942Z","shell.execute_reply.started":"2021-12-31T02:23:22.013995Z","shell.execute_reply":"2021-12-31T02:23:23.031085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y13)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:23.033154Z","iopub.execute_input":"2021-12-31T02:23:23.033398Z","iopub.status.idle":"2021-12-31T02:23:24.069068Z","shell.execute_reply.started":"2021-12-31T02:23:23.033368Z","shell.execute_reply":"2021-12-31T02:23:24.068205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rcParams['figure.figsize'] = 20,5\nplt.plot(item1.ds, item1.y14)","metadata":{"execution":{"iopub.status.busy":"2021-12-31T02:23:24.070211Z","iopub.execute_input":"2021-12-31T02:23:24.070427Z","iopub.status.idle":"2021-12-31T02:23:24.914576Z","shell.execute_reply.started":"2021-12-31T02:23:24.0704Z","shell.execute_reply":"2021-12-31T02:23:24.913516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}