{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport time \nimport datetime\nfrom sklearn.svm import SVR\nimport gresearch_crypto","metadata":{"papermill":{"duration":2.547093,"end_time":"2022-01-22T11:42:21.94375","exception":false,"start_time":"2022-01-22T11:42:19.396657","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:57:34.916535Z","iopub.execute_input":"2022-06-09T17:57:34.916988Z","iopub.status.idle":"2022-06-09T17:57:34.923903Z","shell.execute_reply.started":"2022-06-09T17:57:34.916955Z","shell.execute_reply":"2022-06-09T17:57:34.922912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_strict(file_name='/kaggle/input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    train_df = df[df['datetime'] < '2021-06-13 00:00:00']\n    test_df = df[df['datetime'] >= '2021-06-13 00:00:00']\n    return train_df, test_df\n\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return corr","metadata":{"papermill":{"duration":0.02926,"end_time":"2022-01-22T11:42:21.990291","exception":false,"start_time":"2022-01-22T11:42:21.961031","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:29:50.590352Z","iopub.execute_input":"2022-06-09T17:29:50.59114Z","iopub.status.idle":"2022-06-09T17:29:50.604371Z","shell.execute_reply.started":"2022-06-09T17:29:50.591093Z","shell.execute_reply":"2022-06-09T17:29:50.602951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"papermill":{"duration":0.024291,"end_time":"2022-01-22T11:42:22.030943","exception":false,"start_time":"2022-01-22T11:42:22.006652","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:29:50.606357Z","iopub.execute_input":"2022-06-09T17:29:50.607156Z","iopub.status.idle":"2022-06-09T17:29:50.616814Z","shell.execute_reply.started":"2022-06-09T17:29:50.607116Z","shell.execute_reply":"2022-06-09T17:29:50.615771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train, df_test = read_csv_strict()","metadata":{"papermill":{"duration":69.085928,"end_time":"2022-01-22T11:43:31.133499","exception":false,"start_time":"2022-01-22T11:42:22.047571","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:29:50.619993Z","iopub.execute_input":"2022-06-09T17:29:50.620838Z","iopub.status.idle":"2022-06-09T17:30:56.24897Z","shell.execute_reply.started":"2022-06-09T17:29:50.620788Z","shell.execute_reply":"2022-06-09T17:30:56.247707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"papermill":{"duration":0.042612,"end_time":"2022-01-22T11:43:31.193321","exception":false,"start_time":"2022-01-22T11:43:31.150709","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:30:56.250785Z","iopub.execute_input":"2022-06-09T17:30:56.251222Z","iopub.status.idle":"2022-06-09T17:30:56.270423Z","shell.execute_reply.started":"2022-06-09T17:30:56.251186Z","shell.execute_reply":"2022-06-09T17:30:56.269108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.iloc[2,:]","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:30:56.272165Z","iopub.execute_input":"2022-06-09T17:30:56.272895Z","iopub.status.idle":"2022-06-09T17:30:56.282712Z","shell.execute_reply.started":"2022-06-09T17:30:56.272849Z","shell.execute_reply":"2022-06-09T17:30:56.281496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n    df_feat = df[['timestamp','Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','datetime']].copy()\n    df_feat['output']=pd.np.nan\n    for i in range (len(df_feat)-1):\n        df_feat['output'].iloc[i]=df_feat['Close'].iloc[i+1]\n    \n    return df_feat","metadata":{"papermill":{"duration":0.030283,"end_time":"2022-01-22T11:43:31.240487","exception":false,"start_time":"2022-01-22T11:43:31.210204","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:30:56.284209Z","iopub.execute_input":"2022-06-09T17:30:56.284891Z","iopub.status.idle":"2022-06-09T17:30:56.29521Z","shell.execute_reply.started":"2022-06-09T17:30:56.284844Z","shell.execute_reply":"2022-06-09T17:30:56.294053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df_train[df_train[\"Asset_ID\"] == 0]\ndf_proc = get_features(df)","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:30:56.296397Z","iopub.execute_input":"2022-06-09T17:30:56.297299Z","iopub.status.idle":"2022-06-09T17:41:15.74761Z","shell.execute_reply.started":"2022-06-09T17:30:56.297242Z","shell.execute_reply":"2022-06-09T17:41:15.745904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_proc.columns","metadata":{"papermill":{"duration":0.035327,"end_time":"2022-01-22T11:43:31.293017","exception":false,"start_time":"2022-01-22T11:43:31.25769","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T17:41:15.750345Z","iopub.execute_input":"2022-06-09T17:41:15.750985Z","iopub.status.idle":"2022-06-09T17:41:15.75872Z","shell.execute_reply.started":"2022-06-09T17:41:15.750926Z","shell.execute_reply":"2022-06-09T17:41:15.757417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom math import sqrt","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:41:15.762511Z","iopub.execute_input":"2022-06-09T17:41:15.763051Z","iopub.status.idle":"2022-06-09T17:41:15.780392Z","shell.execute_reply.started":"2022-06-09T17:41:15.762996Z","shell.execute_reply":"2022-06-09T17:41:15.779211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_proc.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:55:40.160142Z","iopub.execute_input":"2022-06-09T17:55:40.161125Z","iopub.status.idle":"2022-06-09T17:55:40.181447Z","shell.execute_reply.started":"2022-06-09T17:55:40.161077Z","shell.execute_reply":"2022-06-09T17:55:40.180299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\nvalid_window = [totimestamp(\"12/06/2021\")]","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:57:38.359845Z","iopub.execute_input":"2022-06-09T17:57:38.360294Z","iopub.status.idle":"2022-06-09T17:57:38.366734Z","shell.execute_reply.started":"2022-06-09T17:57:38.360243Z","shell.execute_reply":"2022-06-09T17:57:38.365834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=df_proc[df_proc['timestamp']<valid_window[0]]","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:00:02.417937Z","iopub.execute_input":"2022-06-09T18:00:02.418936Z","iopub.status.idle":"2022-06-09T18:00:02.583678Z","shell.execute_reply.started":"2022-06-09T18:00:02.418893Z","shell.execute_reply":"2022-06-09T18:00:02.582527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:00:18.190139Z","iopub.execute_input":"2022-06-09T18:00:18.190616Z","iopub.status.idle":"2022-06-09T18:00:18.212507Z","shell.execute_reply.started":"2022-06-09T18:00:18.190577Z","shell.execute_reply":"2022-06-09T18:00:18.211523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid=df_proc[df_proc['timestamp']>=valid_window[0]]","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:00:09.372566Z","iopub.execute_input":"2022-06-09T18:00:09.373018Z","iopub.status.idle":"2022-06-09T18:00:09.381098Z","shell.execute_reply.started":"2022-06-09T18:00:09.372983Z","shell.execute_reply":"2022-06-09T18:00:09.380172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:00:34.957407Z","iopub.execute_input":"2022-06-09T18:00:34.957863Z","iopub.status.idle":"2022-06-09T18:00:34.980878Z","shell.execute_reply.started":"2022-06-09T18:00:34.957827Z","shell.execute_reply":"2022-06-09T18:00:34.979778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\nvalid=valid.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n\nX_train=train.drop([\"datetime\",'timestamp','output'],axis=1)\ny_train=train['output']\nX_test=valid.drop([\"datetime\",'timestamp','output'],axis=1)\ny_test=valid['output']\n\n    # TODO: Try different models here!\nmodel = LGBMRegressor()\nmodel.fit(X_train, y_train)\npred = model.predict(X_test)\n\nMSE =sqrt(mean_squared_error(y_test,pred))\nprint('MSE: %.3f' % MSE)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:06:34.0489Z","iopub.execute_input":"2022-06-09T18:06:34.049373Z","iopub.status.idle":"2022-06-09T18:06:38.264938Z","shell.execute_reply.started":"2022-06-09T18:06:34.049337Z","shell.execute_reply":"2022-06-09T18:06:38.26409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid['output']=pred","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:07:33.576403Z","iopub.execute_input":"2022-06-09T18:07:33.576842Z","iopub.status.idle":"2022-06-09T18:07:33.581616Z","shell.execute_reply.started":"2022-06-09T18:07:33.576809Z","shell.execute_reply":"2022-06-09T18:07:33.580391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid['Close']=y_test","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:07:31.849194Z","iopub.execute_input":"2022-06-09T18:07:31.849739Z","iopub.status.idle":"2022-06-09T18:07:31.855959Z","shell.execute_reply.started":"2022-06-09T18:07:31.849696Z","shell.execute_reply":"2022-06-09T18:07:31.854736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:07:37.360501Z","iopub.execute_input":"2022-06-09T18:07:37.36098Z","iopub.status.idle":"2022-06-09T18:07:37.381369Z","shell.execute_reply.started":"2022-06-09T18:07:37.36094Z","shell.execute_reply":"2022-06-09T18:07:37.380482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc = valid.set_index(\"datetime\")\nbtc_mini = btc.iloc[-200:]\n","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:09:39.023419Z","iopub.execute_input":"2022-06-09T18:09:39.023866Z","iopub.status.idle":"2022-06-09T18:09:39.031696Z","shell.execute_reply.started":"2022-06-09T18:09:39.023829Z","shell.execute_reply":"2022-06-09T18:09:39.030579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"btc_mini","metadata":{"execution":{"iopub.status.busy":"2022-06-09T17:53:25.729235Z","iopub.execute_input":"2022-06-09T17:53:25.729693Z","iopub.status.idle":"2022-06-09T17:53:25.757463Z","shell.execute_reply.started":"2022-06-09T17:53:25.729656Z","shell.execute_reply":"2022-06-09T17:53:25.756466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['Close'])])\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-09T18:09:42.550959Z","iopub.execute_input":"2022-06-09T18:09:42.551489Z","iopub.status.idle":"2022-06-09T18:09:42.613754Z","shell.execute_reply.started":"2022-06-09T18:09:42.551447Z","shell.execute_reply":"2022-06-09T18:09:42.612968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.Figure(data=[go.Candlestick(x=btc_mini.index, open=btc_mini['Open'], high=btc_mini['High'], low=btc_mini['Low'], close=btc_mini['output'])])\nfig.show()","metadata":{"papermill":{"duration":0.024638,"end_time":"2022-01-22T11:53:31.246073","exception":false,"start_time":"2022-01-22T11:53:31.221435","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-06-09T18:09:47.252871Z","iopub.execute_input":"2022-06-09T18:09:47.253276Z","iopub.status.idle":"2022-06-09T18:09:47.310018Z","shell.execute_reply.started":"2022-06-09T18:09:47.253245Z","shell.execute_reply":"2022-06-09T18:09:47.309234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}