{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom sklearn.linear_model import LinearRegression\nimport gresearch_crypto\n\n\nTRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\n\ndef read_csv_strict(file_name='../input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    df['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\n    #df = df[((df['datetime'] < '2019-03-13 00:00:00') & (df['datetime'] > '2019-02-13 00:00:00')) | ((df['datetime'] < '2019-12-13 00:00:00') & (df['datetime'] > '2019-06-13 00:00:00')) |((df['datetime'] < '2020-03-13 00:00:00') & (df['datetime'] > '2020-02-13 00:00:00')) |((df['datetime'] < '2020-08-13 00:00:00') & (df['datetime'] > '2020-07-13 00:00:00')) |((df['datetime'] < '2021-03-13 00:00:00') & (df['datetime'] > '2021-01-13 00:00:00')) |((df['datetime'] < '2021-05-13 00:00:00') & (df['datetime'] > '2021-04-13 00:00:00'))]\n    df = df[df['datetime'] < '2021-06-13 00:00:00']\n    df = df[df['datetime'] > '2020-06-13 00:00:00']\n    return df","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-06T01:05:26.422606Z","iopub.execute_input":"2022-01-06T01:05:26.422911Z","iopub.status.idle":"2022-01-06T01:05:28.310365Z","shell.execute_reply.started":"2022-01-06T01:05:26.422825Z","shell.execute_reply":"2022-01-06T01:05:28.309671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = read_csv_strict()\ndf_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:05:28.311938Z","iopub.execute_input":"2022-01-06T01:05:28.312586Z","iopub.status.idle":"2022-01-06T01:06:24.287053Z","shell.execute_reply.started":"2022-01-06T01:05:28.312551Z","shell.execute_reply":"2022-01-06T01:06:24.286286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Volume','Asset_ID']].copy()\n    df_feat[\"day\"] = df['datetime'].dt.day\n    df_feat[\"week\"] = df['datetime'].dt.month\n    df_feat[\"weekday\"] = df['datetime'].dt.weekday\n\n    df_feat['Upper_Shadow'] = upper_shadow(df)\n    df_feat['Lower_Shadow'] = lower_shadow(df)\n    df_feat[\"high_div_low\"] = df[\"High\"] / df[\"Low\"]\n    df_feat[\"open_sub_close\"] = df[\"Open\"] - df[\"Close\"]\n    #df_feat=(df_feat-df_feat.min())/(df_feat.max()-df_feat.min())\n\n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:06:24.288413Z","iopub.execute_input":"2022-01-06T01:06:24.288649Z","iopub.status.idle":"2022-01-06T01:06:24.296467Z","shell.execute_reply.started":"2022-01-06T01:06:24.288623Z","shell.execute_reply":"2022-01-06T01:06:24.294958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = {\n    \"objective\": \"regression\",\n    \"n_estimators\" : 5000,     # <-- (9) change from 200 to 500\n    \"num_leaves\" : 800,       # <-- (10) Added parameter\n    \"learning_rate\" : 0.07,   # <-- (10) Added parameter\n    \"random_seed\" : 1234}","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:06:24.297874Z","iopub.execute_input":"2022-01-06T01:06:24.298120Z","iopub.status.idle":"2022-01-06T01:06:24.310202Z","shell.execute_reply.started":"2022-01-06T01:06:24.298092Z","shell.execute_reply":"2022-01-06T01:06:24.309064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(**best_params)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:06:24.311994Z","iopub.execute_input":"2022-01-06T01:06:24.312238Z","iopub.status.idle":"2022-01-06T01:06:24.326083Z","shell.execute_reply.started":"2022-01-06T01:06:24.312208Z","shell.execute_reply":"2022-01-06T01:06:24.325151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:06:24.327274Z","iopub.execute_input":"2022-01-06T01:06:24.328577Z","iopub.status.idle":"2022-01-06T01:53:49.148630Z","shell.execute_reply.started":"2022-01-06T01:06:24.328536Z","shell.execute_reply":"2022-01-06T01:53:49.147515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:53:49.150571Z","iopub.execute_input":"2022-01-06T01:53:49.150904Z","iopub.status.idle":"2022-01-06T01:53:49.157043Z","shell.execute_reply.started":"2022-01-06T01:53:49.150849Z","shell.execute_reply":"2022-01-06T01:53:49.156525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:53:49.158248Z","iopub.execute_input":"2022-01-06T01:53:49.158622Z","iopub.status.idle":"2022-01-06T01:53:49.175657Z","shell.execute_reply.started":"2022-01-06T01:53:49.158580Z","shell.execute_reply":"2022-01-06T01:53:49.174991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, (df_test, df_pred) in enumerate(iter_test):\n    df_test['datetime'] = pd.to_datetime(df_test['timestamp'], unit='s')\n    df = df_test[['Count', 'Volume','Asset_ID','row_id']].copy()\n    df[\"day\"] = df_test['datetime'].dt.day\n    df[\"week\"] = df_test['datetime'].dt.month\n    df[\"weekday\"] = df_test['datetime'].dt.weekday\n\n    df['Upper_Shadow'] = upper_shadow(df_test)\n    df['Lower_Shadow'] = lower_shadow(df_test)\n    df[\"high_div_low\"] = df_test[\"High\"] / df_test[\"Low\"]\n    df[\"open_sub_close\"] = df_test[\"Open\"] - df_test[\"Close\"]\n    df.head()\n    for j,row in df.iterrows():\n        model = models[int(row['Asset_ID'])]\n        rows=row.to_frame().T\n        x_input=rows[['Count', 'Volume','Asset_ID',\"day\",\"week\",\"weekday\",'Upper_Shadow','Lower_Shadow',\"high_div_low\",\"open_sub_close\"]].copy()\n        y_pred = model.predict(x_input)\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    \n    \n    '''\n    for i in range(14):\n        model = models[i]\n        x_test=df[df[\"Asset_ID\"] == i]\n        x_test = x_test[['Count', 'Volume','Asset_ID',\"day\",\"week\",\"weekday\",'Upper_Shadow','Lower_Shadow',\"high_div_low\",\"open_sub_close\"]].copy()\n        x_test.head()\n        y_pred = model.predict(x_test)\n        print(y_pred)\n        for j,row in x_test.iterrows():\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred[0]\n    '''\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T01:53:49.177186Z","iopub.execute_input":"2022-01-06T01:53:49.177616Z","iopub.status.idle":"2022-01-06T01:53:49.782322Z","shell.execute_reply.started":"2022-01-06T01:53:49.177574Z","shell.execute_reply":"2022-01-06T01:53:49.781460Z"},"trusted":true},"execution_count":null,"outputs":[]}]}