{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-31T18:57:34.776489Z","iopub.execute_input":"2022-01-31T18:57:34.777457Z","iopub.status.idle":"2022-01-31T18:57:34.815111Z","shell.execute_reply.started":"2022-01-31T18:57:34.777286Z","shell.execute_reply":"2022-01-31T18:57:34.814134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import traceback\nimport numpy as np\nimport datatable as dt\nimport gresearch_crypto\nimport pandas as pd\nimport datatable as dt\ndataset = dt.fread('../input/g-research-crypto-forecasting/train.csv').to_pandas()\ndataset.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-31T18:57:46.470137Z","iopub.execute_input":"2022-01-31T18:57:46.470679Z","iopub.status.idle":"2022-01-31T18:58:22.429472Z","shell.execute_reply.started":"2022-01-31T18:57:46.470629Z","shell.execute_reply":"2022-01-31T18:58:22.428282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assetDetails = dt.fread('/kaggle/input/g-research-crypto-forecasting/asset_details.csv').to_pandas().sort_values('Asset_ID')\nassetDetails","metadata":{"execution":{"iopub.status.busy":"2022-01-31T18:58:29.347777Z","iopub.execute_input":"2022-01-31T18:58:29.348130Z","iopub.status.idle":"2022-01-31T18:58:29.382744Z","shell.execute_reply.started":"2022-01-31T18:58:29.348096Z","shell.execute_reply":"2022-01-31T18:58:29.381775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upperShadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lowerShadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef getFeatures(df):\n    modified_df = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', \n                      'VWAP']].copy()\n    modified_df['Upper Shadow'] = upperShadow(modified_df)\n    modified_df['Lower Shadow'] = lowerShadow(modified_df)\n    return modified_df","metadata":{"execution":{"iopub.status.busy":"2022-01-31T18:58:31.214662Z","iopub.execute_input":"2022-01-31T18:58:31.215000Z","iopub.status.idle":"2022-01-31T18:58:31.223730Z","shell.execute_reply.started":"2022-01-31T18:58:31.214967Z","shell.execute_reply":"2022-01-31T18:58:31.222437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb\n\ndef get_Xy_and_model_for_asset(dfTrain, assetId):\n    df = dfTrain[dfTrain['Asset_ID'] == assetId]\n    df_proc = getFeatures(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how='any')\n    \n    X = df_proc.drop('y', axis=1)\n    y = df_proc['y']\n    \n    model = xgb.XGBRegressor(\n        n_estimators=500,\n        max_depth=11,\n        learning_rate=0.06,\n        subsample=0.9,\n        colsample_bytree=0.75,\n        missing=-999,\n        random_state=2022\n        #tree_method='gpu_hist'\n    )\n    \n    model.fit(X,y)\n    return X, y, model\n","metadata":{"execution":{"iopub.status.busy":"2022-01-31T08:34:29.815717Z","iopub.execute_input":"2022-01-31T08:34:29.816297Z","iopub.status.idle":"2022-01-31T08:34:29.909785Z","shell.execute_reply.started":"2022-01-31T08:34:29.816261Z","shell.execute_reply":"2022-01-31T08:34:29.909105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs ={}\nys ={}\nmodels ={}\n\nfor assetId, assetName in zip(assetDetails['Asset_ID'], \n                              assetDetails['Asset_Name']):\n    print(f\"Training model for {assetName:<16} (ID={assetId:<2})\")\n    try:\n        X, y, model = get_Xy_and_model_for_asset(dataset, assetId)\n        Xs[assetId], ys[assetId], models[assetId] = X, y, model\n    except:\n        Xs[assetId], ys[assetId], models[assetId] = None, None, None\n        \n        ","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:36:50.231368Z","iopub.status.idle":"2022-01-31T04:36:50.231705Z","shell.execute_reply.started":"2022-01-31T04:36:50.231516Z","shell.execute_reply":"2022-01-31T04:36:50.231535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = getFeatures(dataset.iloc[1])\ny_pred = models[0].predict(pd.DataFrame([x]))\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:36:50.232605Z","iopub.status.idle":"2022-01-31T04:36:50.232897Z","shell.execute_reply.started":"2022-01-31T04:36:50.23275Z","shell.execute_reply":"2022-01-31T04:36:50.232768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        \n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = getFeatures(row)\n                y_red = model.predict(pd.DataFrame([x_test]))[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n        \n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T04:36:50.234036Z","iopub.status.idle":"2022-01-31T04:36:50.234299Z","shell.execute_reply.started":"2022-01-31T04:36:50.234151Z","shell.execute_reply":"2022-01-31T04:36:50.234167Z"},"trusted":true},"execution_count":null,"outputs":[]}]}