{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport time\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport gresearch_crypto\nimport gc\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\ndef read_csv_strict(file_name='/kaggle/input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df[df['datetime'] > '2019-12-1 00:00:00' ].set_index('timestamp')\n    df = df[df['datetime'] < '2021-06-13 00:00:00' ]\n    \n    return df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-22T21:59:39.980960Z","iopub.execute_input":"2022-01-22T21:59:39.981299Z","iopub.status.idle":"2022-01-22T21:59:39.991844Z","shell.execute_reply.started":"2022-01-22T21:59:39.981243Z","shell.execute_reply":"2022-01-22T21:59:39.990839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()\nasset=[]\nfor i in range(14):\n    asset.append(df_train[df_train['Asset_ID']==i])\ndel df_train","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:59:39.993323Z","iopub.execute_input":"2022-01-22T21:59:39.993592Z","iopub.status.idle":"2022-01-22T22:00:35.087990Z","shell.execute_reply.started":"2022-01-22T21:59:39.993559Z","shell.execute_reply":"2022-01-22T22:00:35.087006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"FE","metadata":{}},{"cell_type":"code","source":"def log_return(series, periods=1):\n    return np.log1p(series).diff(periods=periods)\ndef get_features(data):\n    \n    data['Upper_Shadow'] = data['High'] - np.maximum(data['Close'], data['Open'])\n    data['Lower_Shadow'] = np.minimum(data['Close'], data['Open']) - data['Low']\n    \n    \n    data['volume2count'] = data['Volume'] / (data['Count'] + 1)\n    \n    data['Low_15min']=(pd.concat([data['Low'].shift(i) for i in range(16)],axis=1)).min(axis=1)\n    data['High_15min']=(pd.concat([data['High'].shift(i) for i in range(16)],axis=1)).max(axis=1)\n\n    data['Upper_Shadow_15min'] = data['High_15min'] - np.maximum(data['Close'], data['Open'])\n    data['Lower_Shadow_15min'] = np.minimum(data['Close'], data['Open']) - data['Low_15min']\n    \n    for i in [1,2,15,16]:\n        data['Target-'+str(i)]=data['Target'].shift(i)\n    \n    for i in [1,15,45,60]:\n        data['lr'+str(i)]=log_return(data['Close'], periods=i)\n        data['lrv'+str(i)]=log_return(data['VWAP'], periods=i)\n    \n    data['Close_position']=(data['Close']-data['Low'])/(data['High']-data['Low']+1)\n    data['Close_position_15min']=(data['Close']-data['Low_15min'])/(data['High_15min']-data['Low_15min']+1)\n    \n    data['Open_position']=(data['Open']-data['Low'])/(data['High']-data['Low']+1)\n    data['Open_position_15min']=(data['Open']-data['Low_15min'])/(data['High_15min']-data['Low_15min']+1)   \n    \n    data['h/h15']=(data['High']/data['High_15min']).transform(np.log1p)\n    data['l/l15']=(data['Low']/data['Low_15min']).transform(np.log1p)\n    for c in ['High','Low','VWAP','High_15min','Low_15min']:\n        data['Close/'+c]=(data['Close']/data[c]).transform(np.log1p)\n    for c in ['Open','High','Low','Close','VWAP','High_15min','Low_15min']:\n        data[c]=data[c].transform(np.log1p)\n    \n    return 0","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:00:35.089367Z","iopub.execute_input":"2022-01-22T22:00:35.089624Z","iopub.status.idle":"2022-01-22T22:00:35.113755Z","shell.execute_reply.started":"2022-01-22T22:00:35.089593Z","shell.execute_reply":"2022-01-22T22:00:35.112038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for a in asset :\n    get_features(a)\n    a.replace(-np.inf, np.nan,inplace=True)\n    a.replace(np.inf, np.nan,inplace=True)\n    a.dropna(inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:00:35.115516Z","iopub.execute_input":"2022-01-22T22:00:35.115912Z","iopub.status.idle":"2022-01-22T22:00:51.815792Z","shell.execute_reply.started":"2022-01-22T22:00:35.115869Z","shell.execute_reply":"2022-01-22T22:00:51.814793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features=asset[1].columns.difference(['Asset_ID','Target','datetime'])","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:00:51.817258Z","iopub.execute_input":"2022-01-22T22:00:51.817515Z","iopub.status.idle":"2022-01-22T22:00:51.824050Z","shell.execute_reply.started":"2022-01-22T22:00:51.817476Z","shell.execute_reply":"2022-01-22T22:00:51.823053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x=[]\nfor i in range (14):\n    x.append( asset[i].loc[:, features].values)\n    x[i] = StandardScaler().fit_transform(x[i])","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:00:51.825672Z","iopub.execute_input":"2022-01-22T22:00:51.826186Z","iopub.status.idle":"2022-01-22T22:01:04.391058Z","shell.execute_reply.started":"2022-01-22T22:00:51.826137Z","shell.execute_reply":"2022-01-22T22:01:04.390097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"principal_df=[]\nfor k in range(14):\n    pca_df = PCA(0.95)\n    principalComponents_df = pca_df.fit_transform(x[k])\n    principal_df .append( pd.DataFrame(data = principalComponents_df\n                 , columns = ['feature '+str(i) for i in range(1,principalComponents_df.shape[1]+1)]))","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:01:04.394262Z","iopub.execute_input":"2022-01-22T22:01:04.394675Z","iopub.status.idle":"2022-01-22T22:01:32.808218Z","shell.execute_reply.started":"2022-01-22T22:01:04.394627Z","shell.execute_reply":"2022-01-22T22:01:32.807316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for k in range(14):\n    principal_df[k]['timestamp']=asset[k].reset_index()['timestamp']\n    principal_df[k]=principal_df[k].set_index('timestamp')\ndel x","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:01:32.809637Z","iopub.execute_input":"2022-01-22T22:01:32.809870Z","iopub.status.idle":"2022-01-22T22:01:34.694197Z","shell.execute_reply.started":"2022-01-22T22:01:32.809841Z","shell.execute_reply":"2022-01-22T22:01:34.693036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=[]\nfor i in range(14):\n    y.append(asset[i].Target)","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:01:34.696058Z","iopub.execute_input":"2022-01-22T22:01:34.696425Z","iopub.status.idle":"2022-01-22T22:01:34.702772Z","shell.execute_reply.started":"2022-01-22T22:01:34.696381Z","shell.execute_reply":"2022-01-22T22:01:34.701795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models=[]\nfor i in range(14):\n    models.append(LinearRegression())","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:01:34.704388Z","iopub.execute_input":"2022-01-22T22:01:34.704698Z","iopub.status.idle":"2022-01-22T22:01:34.717485Z","shell.execute_reply.started":"2022-01-22T22:01:34.704668Z","shell.execute_reply":"2022-01-22T22:01:34.716252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfor i in range(14):\n    models[i].fit(principal_df[i][:200000],y[i][:200000])\n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:02:04.073060Z","iopub.execute_input":"2022-01-22T22:02:04.074026Z","iopub.status.idle":"2022-01-22T22:02:05.378543Z","shell.execute_reply.started":"2022-01-22T22:02:04.073949Z","shell.execute_reply":"2022-01-22T22:02:05.377540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds=[]\nfor i in range(14):\n    pred=(models[i].predict(principal_df[i][200000:210000]))\n    preds.append(pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:02:29.607271Z","iopub.execute_input":"2022-01-22T22:02:29.607614Z","iopub.status.idle":"2022-01-22T22:02:29.664308Z","shell.execute_reply.started":"2022-01-22T22:02:29.607577Z","shell.execute_reply":"2022-01-22T22:02:29.663210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(14):\n    pred=pd.Series(preds[i].reshape(10000,))\n    true=pd.Series(y[i][200000:210000].reset_index().Target)\n    print('for asset'+str(i))\n    print(pd.DataFrame({'pred':pred,'true':true}).corr().iloc[1,0])","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:02:40.599335Z","iopub.execute_input":"2022-01-22T22:02:40.600720Z","iopub.status.idle":"2022-01-22T22:02:40.651542Z","shell.execute_reply.started":"2022-01-22T22:02:40.600642Z","shell.execute_reply":"2022-01-22T22:02:40.650546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ni=500\nx=240000\nplt.plot(principal_df[1][x:x+i].index,y[1][x:x+i])\nplt.plot(principal_df[1][x:x+i].index, models[1].predict(principal_df[1][x:x+i]))","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:05:18.537573Z","iopub.execute_input":"2022-01-22T22:05:18.537935Z","iopub.status.idle":"2022-01-22T22:05:19.177200Z","shell.execute_reply.started":"2022-01-22T22:05:18.537899Z","shell.execute_reply":"2022-01-22T22:05:19.175976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i=500\nx=240000\nplt.plot(principal_df[0][x:x+i].index,y[0][x:x+i])\nplt.plot(principal_df[0][x:x+i].index, models[0].predict(principal_df[0][x:x+i]))","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:08:40.143903Z","iopub.execute_input":"2022-01-22T22:08:40.144377Z","iopub.status.idle":"2022-01-22T22:08:40.398840Z","shell.execute_reply.started":"2022-01-22T22:08:40.144337Z","shell.execute_reply":"2022-01-22T22:08:40.397936Z"},"trusted":true},"execution_count":null,"outputs":[]}]}