{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\n\ndef read_csv_strict(file_name='../input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df[df['datetime'] < '2021-06-13 00:00:00']\n    return df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-26T06:40:57.030356Z","iopub.execute_input":"2021-12-26T06:40:57.031334Z","iopub.status.idle":"2021-12-26T06:40:57.05957Z","shell.execute_reply.started":"2021-12-26T06:40:57.031202Z","shell.execute_reply":"2021-12-26T06:40:57.058908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = read_csv_strict()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:40:57.061101Z","iopub.execute_input":"2021-12-26T06:40:57.061513Z","iopub.status.idle":"2021-12-26T06:42:02.038933Z","shell.execute_reply.started":"2021-12-26T06:40:57.061482Z","shell.execute_reply":"2021-12-26T06:42:02.038114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import data","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport gresearch_crypto\n\nTRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\n\ndef read_csv_strict(file_name='../input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    df = df[df['datetime'] < '2021-06-13 00:00:00']\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:42:02.040051Z","iopub.execute_input":"2021-12-26T06:42:02.04031Z","iopub.status.idle":"2021-12-26T06:42:04.079904Z","shell.execute_reply.started":"2021-12-26T06:42:02.04028Z","shell.execute_reply":"2021-12-26T06:42:04.079256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:42:04.081563Z","iopub.execute_input":"2021-12-26T06:42:04.082151Z","iopub.status.idle":"2021-12-26T06:42:49.906975Z","shell.execute_reply.started":"2021-12-26T06:42:04.082104Z","shell.execute_reply":"2021-12-26T06:42:49.905554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:42:49.912824Z","iopub.execute_input":"2021-12-26T06:42:49.91366Z","iopub.status.idle":"2021-12-26T06:42:49.952979Z","shell.execute_reply.started":"2021-12-26T06:42:49.913602Z","shell.execute_reply":"2021-12-26T06:42:49.952108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"def upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# Build features from original dataframe\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    return df_feat\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    model = LGBMRegressor(n_estimators=10)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:42:49.954132Z","iopub.execute_input":"2021-12-26T06:42:49.95437Z","iopub.status.idle":"2021-12-26T06:42:49.964416Z","shell.execute_reply.started":"2021-12-26T06:42:49.954341Z","shell.execute_reply":"2021-12-26T06:42:49.963572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loop over assets","metadata":{}},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:42:49.965811Z","iopub.execute_input":"2021-12-26T06:42:49.966143Z","iopub.status.idle":"2021-12-26T06:43:11.977607Z","shell.execute_reply.started":"2021-12-26T06:42:49.966106Z","shell.execute_reply":"2021-12-26T06:43:11.976935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = get_features(df_train.iloc[1])\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:43:11.981677Z","iopub.execute_input":"2021-12-26T06:43:11.984059Z","iopub.status.idle":"2021-12-26T06:43:11.997226Z","shell.execute_reply.started":"2021-12-26T06:43:11.984012Z","shell.execute_reply":"2021-12-26T06:43:11.996586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = get_features(row)\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-26T06:43:12.000829Z","iopub.execute_input":"2021-12-26T06:43:12.003043Z","iopub.status.idle":"2021-12-26T06:43:12.221365Z","shell.execute_reply.started":"2021-12-26T06:43:12.003001Z","shell.execute_reply":"2021-12-26T06:43:12.220681Z"},"trusted":true},"execution_count":null,"outputs":[]}]}