{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-05T13:31:41.519612Z","iopub.execute_input":"2022-01-05T13:31:41.520659Z","iopub.status.idle":"2022-01-05T13:31:41.554664Z","shell.execute_reply.started":"2022-01-05T13:31:41.520535Z","shell.execute_reply":"2022-01-05T13:31:41.55402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\npd.options.mode.chained_assignment = None\nimport gc\nfrom datetime import datetime\n\ndirectory = '/kaggle/input/g-research-crypto-forecasting/'\nfile_path = os.path.join(directory, 'train.csv')\ndtypes = {\n    'timestamp': np.int64,\n    'Asset_ID': np.int8,\n     'Count': np.int32,\n     'Open': np.float64,\n     'High': np.float64,\n     'Low': np.float64,\n    'Close': np.float64,\n     'Volume': np.float64,\n     'VWAP': np.float64,\n    'Target': np.float64,\n}\n\ndata = pd.read_csv(file_path, dtype=dtypes, usecols=list(dtypes.keys()))\ndata.set_index('timestamp', inplace = True)\ndata_recent = data.loc[datetime.timestamp(datetime(2021,1,1)):datetime.timestamp(datetime(2021,6,13))]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:31:41.556029Z","iopub.execute_input":"2022-01-05T13:31:41.556478Z","iopub.status.idle":"2022-01-05T13:32:37.353224Z","shell.execute_reply.started":"2022-01-05T13:31:41.556446Z","shell.execute_reply":"2022-01-05T13:32:37.352327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import GradientBoostingRegressor\n\ndef prepare_df(df, with_target = True):\n    df_feat = df[['Count', 'Close', 'Volume']].copy()\n    df_feat.columns = [col_name + '15' for col_name in df_feat.columns]\n    df_feat = df_feat.diff(15)\n\n    df = pd.concat([df_feat, df], axis = 1) \n    df.dropna(inplace = True)\n\n    X = df[['Count15', 'Close15', 'Volume15']]\n    if with_target:\n        y = df['Target']\n    else:\n        y = None\n    return X, y, df.index\n\ndef get_model(df_train):\n    X_train, y_train, idx = prepare_df(df_train)\n    model = GradientBoostingRegressor(n_estimators = 10)\n    model.fit(X_train, y_train)\n    return model\n\ndef make_prediction(df_pred, model):\n    X_train, _, idx = prepare_df(df_pred, with_target = False)\n    y_pred = model.predict(X_train)\n    return(idx, y_pred)\n\nids = data_recent.Asset_ID.unique()\nmodels = {}\n\nfor id_ in ids:\n    df_train = data_recent[data_recent.Asset_ID == id_]\n    models[id_] = get_model(df_train)\n    print(id_)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:35:35.448844Z","iopub.execute_input":"2022-01-05T13:35:35.449241Z","iopub.status.idle":"2022-01-05T13:36:23.634961Z","shell.execute_reply.started":"2022-01-05T13:35:35.449192Z","shell.execute_reply":"2022-01-05T13:36:23.633957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data up to the start from the test data\ndata_memory = data_recent.drop(['Target'], axis = 1)\ndata_memory = data_memory.iloc[-10000:-14]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T13:45:53.560234Z","iopub.execute_input":"2022-01-05T13:45:53.560523Z","iopub.status.idle":"2022-01-05T13:45:53.610981Z","shell.execute_reply.started":"2022-01-05T13:45:53.560494Z","shell.execute_reply":"2022-01-05T13:45:53.610252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nfrom datetime import datetime\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nstart_time = datetime.now()\nfor df_test, df_pred in iter_test:\n\n    # add new data and memorize how much data we need to drop from the beginning\n    data_to_drop = len(df_test)\n    data_memory.loc[:, 'row_id'] = np.nan\n\n    # set timestamp index for new test data\n    df_test.set_index('timestamp', inplace = True)\n    data_memory = pd.concat([data_memory, df_test], axis = 0)\n    \n    # for each id ...\n    for id_ in ids:\n\n        # step two, a asset specific id with row_ids for which targets are needed\n        need_pred = df_test[df_test.Asset_ID ==  id_]\n\n        # get asset data for prediction\n        data_memory_id = data_memory[data_memory.Asset_ID == id_]\n        # get predictions from the model\n        idx, y_pred = make_prediction(data_memory_id.drop(['row_id'], axis = 1), models[id_])\n        # make dataframe for asset specific predictions\n        df_pred_id = pd.DataFrame(index = idx, data = y_pred, columns = ['Target'])\n\n        # reduce all asset predictions to the ones we need for that asset\n        need_pred = need_pred.merge(df_pred_id, left_index = True, right_index = True)[['row_id', 'Target']]\n\n        try:\n            all_current_preds = pd.concat([all_current_preds, need_pred], axis = 0)\n        except:\n            all_current_preds = need_pred.copy()\n\n    df_pred = df_pred.drop(['Target'], axis = 1).merge(all_current_preds, on = 'row_id', how = 'left')\n    del all_current_preds\n    #print(df_pred)\n    \n    # Send submissions\n    env.predict(df_pred)\n    \n    data_memory = data_memory.iloc[data_to_drop:, :]\n    data_memory.drop(['row_id'], axis = 1, inplace = True)\n    \ntime_elapsed = datetime.now() - start_time\nprint('Time elapsed (hh:mm:ss.ms) {}'.format(time_elapsed))","metadata":{"execution":{"iopub.status.busy":"2022-01-04T14:12:39.634412Z","iopub.execute_input":"2022-01-04T14:12:39.634832Z","iopub.status.idle":"2022-01-04T14:12:46.233365Z","shell.execute_reply.started":"2022-01-04T14:12:39.634785Z","shell.execute_reply":"2022-01-04T14:12:46.232532Z"},"trusted":true},"execution_count":null,"outputs":[]}]}