{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Submitting Lagged Features via API\n\nIn this notebook we submit a LGBM model with lagged features via the API.\n\nThe API works by providing a single row for each Asset - one timestamp at a time - to prevent using future data in predictions.\n\nIn order to utilise lagged features in our model, we must store the outputs from the API so we can calculate features using past data.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:02:21.487323Z","iopub.execute_input":"2021-12-12T14:02:21.487719Z","iopub.status.idle":"2021-12-12T14:02:23.489641Z","shell.execute_reply.started":"2021-12-12T14:02:21.487623Z","shell.execute_reply":"2021-12-12T14:02:23.487712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:10:44.415553Z","iopub.execute_input":"2021-12-12T14:10:44.415889Z","iopub.status.idle":"2021-12-12T14:11:51.221701Z","shell.execute_reply.started":"2021-12-12T14:10:44.415849Z","shell.execute_reply":"2021-12-12T14:11:51.220352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:11:51.225249Z","iopub.execute_input":"2021-12-12T14:11:51.225609Z","iopub.status.idle":"2021-12-12T14:11:51.261008Z","shell.execute_reply.started":"2021-12-12T14:11:51.225559Z","shell.execute_reply":"2021-12-12T14:11:51.260285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, \n                 asset_id, \n                 train=True):\n    '''\n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    df = df.sort_values('timestamp')\n    if train == True:\n        df_feat = df.copy()\n        # define a train_flg column to split your data into train and validation\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n        df_feat['train_flg'] = np.where(df_feat['timestamp']>=valid_window[0], 0,1)\n        df_feat = df_feat[['timestamp','Asset_ID','Close','Target','train_flg']].copy()\n    else:\n        df = df.sort_values('row_id')\n        df_feat = df[['Asset_ID','Close','row_id']].copy()\n    \n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling(15).mean()/df_feat['Close'] -1\n    df_feat['sma60'] = df_feat['Close'].rolling(60).mean()/df_feat['Close'] -1\n    df_feat['sma240'] = df_feat['Close'].rolling(240).mean()/df_feat['Close'] -1\n    \n    df_feat['return15'] = df_feat['Close']/df_feat['Close'].shift(15) -1\n    df_feat['return60'] = df_feat['Close']/df_feat['Close'].shift(60) -1\n    df_feat['return240'] = df_feat['Close']/df_feat['Close'].shift(240) -1\n    df_feat = df_feat.fillna(0)\n    \n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:11:51.262122Z","iopub.execute_input":"2021-12-12T14:11:51.262371Z","iopub.status.idle":"2021-12-12T14:11:51.276435Z","shell.execute_reply.started":"2021-12-12T14:11:51.262341Z","shell.execute_reply":"2021-12-12T14:11:51.275760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create your feature dataframe for each asset and concatenate\nfeature_df = pd.DataFrame()\nfor i in range(14):\n    feature_df = pd.concat([feature_df,get_features(df_train,i,train=True)])","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:18:02.990812Z","iopub.execute_input":"2021-12-12T14:18:02.991188Z","iopub.status.idle":"2021-12-12T14:18:02.996013Z","shell.execute_reply.started":"2021-12-12T14:18:02.991148Z","shell.execute_reply":"2021-12-12T14:18:02.995239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assign weight column feature dataframe\nfeature_df = pd.merge(feature_df, df_asset_details[['Asset_ID','Weight']], how='left', on=['Asset_ID'])","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:14:31.127604Z","iopub.execute_input":"2021-12-12T14:14:31.128019Z","iopub.status.idle":"2021-12-12T14:14:34.476237Z","shell.execute_reply.started":"2021-12-12T14:14:31.127986Z","shell.execute_reply":"2021-12-12T14:14:34.475313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nfeatures = ['Asset_ID','sma15','sma60','sma240','return15','return60','return240']\ncategoricals = ['Asset_ID']","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:14:34.572154Z","iopub.execute_input":"2021-12-12T14:14:34.572443Z","iopub.status.idle":"2021-12-12T14:14:34.576939Z","shell.execute_reply.started":"2021-12-12T14:14:34.572413Z","shell.execute_reply":"2021-12-12T14:14:34.576197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef weighted_correlation(a, train_data):\n    \n    weights = train_data.add_w.values.flatten()\n    b = train_data.get_label()\n    \n    \n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'eval_wcorr', corr, True","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:14:39.244290Z","iopub.execute_input":"2021-12-12T14:14:39.244831Z","iopub.status.idle":"2021-12-12T14:14:39.253284Z","shell.execute_reply.started":"2021-12-12T14:14:39.244759Z","shell.execute_reply":"2021-12-12T14:14:39.252549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define train and validation weights and datasets\nweights_train = feature_df.query('train_flg == 1')[['Weight']]\nweights_test = feature_df.query('train_flg == 0')[['Weight']]\n\ntrain_dataset = lgb.Dataset(feature_df.query('train_flg == 1')[features], \n                            feature_df.query('train_flg == 1')['Target'].values, \n                            feature_name = features, \n                            categorical_feature= categoricals)\nval_dataset = lgb.Dataset(feature_df.query('train_flg == 0')[features], \n                          feature_df.query('train_flg == 0')['Target'].values, \n                          feature_name = features, \n                          categorical_feature= categoricals)\n\ntrain_dataset.add_w = weights_train\nval_dataset.add_w = weights_test\n\nevals_result = {}\nparams = {'n_estimators': 1000,\n        'objective': 'regression',\n        'metric': 'None',\n        'boosting_type': 'gbdt',\n        'max_depth': -1, \n        'learning_rate': 0.01,\n        'seed': 46,\n        'verbose': -1,\n        }\n\n# train LGBM2\nmodel = lgb.train(params = params,\n                  train_set = train_dataset, \n                  valid_sets = [val_dataset],\n                  early_stopping_rounds=100,\n                  verbose_eval = 10,\n                  feval=weighted_correlation,\n                  evals_result = evals_result \n                 )","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:14:43.646809Z","iopub.execute_input":"2021-12-12T14:14:43.647354Z","iopub.status.idle":"2021-12-12T14:17:43.665858Z","shell.execute_reply.started":"2021-12-12T14:14:43.647299Z","shell.execute_reply":"2021-12-12T14:17:43.665125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Important!","metadata":{}},{"cell_type":"code","source":"# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nmax_lookback = 250","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:27:17.624995Z","iopub.execute_input":"2021-12-12T14:27:17.625397Z","iopub.status.idle":"2021-12-12T14:27:17.630843Z","shell.execute_reply.started":"2021-12-12T14:27:17.625356Z","shell.execute_reply":"2021-12-12T14:27:17.629729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now we will submit via api\n\n- As mentioned by the host here https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/290412 - the api takes 10 minutes to complete when submitted on the full test data with a simple dummy prediction. \n\n- Therefore, any extra logic we include within the api loop with increase the time to completion significantly.\n\n- I have not focused on optimisation of the logic within this loop yet - there are definetly significant improvements you can try for yourself. For example, using numpy arrays instead of pandas dataframes may help.\n\n- For this version - the submission time is roughly 5 hours.","metadata":{}},{"cell_type":"code","source":"start = time.time()\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\n# create dataframe to store data from the api to create lagged features\nhistory = pd.DataFrame()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    \n    # concatenate new api data to history dataframe\n    history = pd.concat([history, df_test[['timestamp','Asset_ID','Close','row_id']]])\n    for j , row in df_test.iterrows():\n        # get features using history dataframe\n        row_features = get_features(history, row['Asset_ID'], train=False)\n        row = row_features.iloc[-1].fillna(0)\n        y_pred = model.predict(row[features])[0]\n\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    \n    # we only want to keep the necessary recent part of our history dataframe, which will depend on your\n    # max_lookback value (your furthest lookback in creating lagged features).\n    history = history.sort_values(by='row_id')\n    history = history.iloc[-(max_lookback*14+100):]\n    \n    # Send submissions\n    env.predict(df_pred)\nstop = time.time()\nprint(stop-start)","metadata":{"execution":{"iopub.status.busy":"2021-12-12T14:27:20.174454Z","iopub.execute_input":"2021-12-12T14:27:20.174882Z","iopub.status.idle":"2021-12-12T14:27:20.857192Z","shell.execute_reply.started":"2021-12-12T14:27:20.174847Z","shell.execute_reply":"2021-12-12T14:27:20.856444Z"},"trusted":true},"execution_count":null,"outputs":[]}]}