{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\nimport xgboost as xgb\nimport traceback\nfrom sklearn.preprocessing import StandardScaler\n\nTRAIN_CSV = '../input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '../input/g-research-crypto-forecasting/asset_details.csv'\nTEST_CSV = '../input/g-research-crypto-forecasting/example_test.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:31:16.637251Z","iopub.execute_input":"2022-01-19T21:31:16.637791Z","iopub.status.idle":"2022-01-19T21:31:19.533400Z","shell.execute_reply.started":"2022-01-19T21:31:16.637717Z","shell.execute_reply":"2022-01-19T21:31:19.532690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:31:20.774056Z","iopub.execute_input":"2022-01-19T21:31:20.774714Z","iopub.status.idle":"2022-01-19T21:32:15.311936Z","shell.execute_reply.started":"2022-01-19T21:31:20.774677Z","shell.execute_reply":"2022-01-19T21:32:15.311144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T20:34:28.514179Z","iopub.execute_input":"2022-01-19T20:34:28.514445Z","iopub.status.idle":"2022-01-19T20:34:29.003991Z","shell.execute_reply.started":"2022-01-19T20:34:28.514415Z","shell.execute_reply":"2022-01-19T20:34:29.003254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T18:53:25.753588Z","iopub.execute_input":"2022-01-19T18:53:25.754309Z","iopub.status.idle":"2022-01-19T18:53:26.164227Z","shell.execute_reply.started":"2022-01-19T18:53:25.754257Z","shell.execute_reply":"2022-01-19T18:53:26.163479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(TEST_CSV)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-19T18:20:03.395324Z","iopub.execute_input":"2022-01-19T18:20:03.395588Z","iopub.status.idle":"2022-01-19T18:20:03.41646Z","shell.execute_reply.started":"2022-01-19T18:20:03.395552Z","shell.execute_reply":"2022-01-19T18:20:03.41571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.replace([np.inf, -np.inf], np.nan)\n\n#df_train.fillna(-999, inplace=True)\ndf_train = df_train.dropna(how=\"any\")","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:32:19.209975Z","iopub.execute_input":"2022-01-19T21:32:19.211921Z","iopub.status.idle":"2022-01-19T21:32:22.803403Z","shell.execute_reply.started":"2022-01-19T21:32:19.211884Z","shell.execute_reply":"2022-01-19T21:32:22.802507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info(show_counts =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T18:54:34.247144Z","iopub.execute_input":"2022-01-19T18:54:34.247427Z","iopub.status.idle":"2022-01-19T18:54:34.724193Z","shell.execute_reply.started":"2022-01-19T18:54:34.247396Z","shell.execute_reply":"2022-01-19T18:54:34.723416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:32:22.805415Z","iopub.execute_input":"2022-01-19T21:32:22.805710Z","iopub.status.idle":"2022-01-19T21:32:22.828430Z","shell.execute_reply.started":"2022-01-19T21:32:22.805671Z","shell.execute_reply":"2022-01-19T21:32:22.827784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n   \n    return df_feat\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    df_proc = get_features(df)\n    \n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace((np.inf, -np.inf), np.nan)\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    model = xgb.XGBRegressor(\n    n_estimators=500,\n    learning_rate=0.05,\n    max_depth=12,\n    subsample=0.9,\n    colsample_bytree=0.7,\n    #colsample_bylevel=0.75,\n    missing=-999,\n    random_state=1111,\n    tree_method='gpu_hist'  \n    )\n    model.fit(X, y)\n    \n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:43:14.587274Z","iopub.execute_input":"2022-01-19T21:43:14.587581Z","iopub.status.idle":"2022-01-19T21:43:14.599783Z","shell.execute_reply.started":"2022-01-19T21:43:14.587548Z","shell.execute_reply":"2022-01-19T21:43:14.599080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model ","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:43:16.694569Z","iopub.execute_input":"2022-01-19T21:43:16.695164Z","iopub.status.idle":"2022-01-19T21:51:29.182687Z","shell.execute_reply.started":"2022-01-19T21:43:16.695128Z","shell.execute_reply":"2022-01-19T21:51:29.181913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nx = get_features(df_train.iloc[1])\n#y_pred = models[0].predict([x])\n#y_pred[0]\ny_pred = models[0].predict(pd.DataFrame([x]))\nprint(y_pred[0])\nprint(df_train.iloc[1])","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:52:06.107248Z","iopub.execute_input":"2022-01-19T21:52:06.107537Z","iopub.status.idle":"2022-01-19T21:52:06.127346Z","shell.execute_reply.started":"2022-01-19T21:52:06.107503Z","shell.execute_reply":"2022-01-19T21:52:06.126288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = get_features(row)\n                y_pred = model.predict(pd.DataFrame([x_test]))[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n        \n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T21:52:34.082817Z","iopub.execute_input":"2022-01-19T21:52:34.083698Z","iopub.status.idle":"2022-01-19T21:52:34.429445Z","shell.execute_reply.started":"2022-01-19T21:52:34.083654Z","shell.execute_reply":"2022-01-19T21:52:34.428913Z"},"trusted":true},"execution_count":null,"outputs":[]}]}