{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Submitting Lagged Features via API\n\nIn this notebook we submit a LGBM model with lagged features via the API.\n\nThe API works by providing a single row for each Asset - one timestamp at a time - to prevent using future data in predictions.\n\nIn order to utilise lagged features in our model, we must store the outputs from the API so we can calculate features using past data.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\nimport gc\nimport traceback\nimport datatable as dt\nimport gresearch_crypto\nfrom tqdm.notebook import tqdm\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import roc_auc_score, mean_absolute_error\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\nDEVICE = 'GPU'\n# CV PARAMS\nFOLDS = 5\nGROUP_GAP = 130\nMAX_TEST_GROUP_SIZE = 180\nMAX_TRAIN_GROUP_SIZE = 280","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:52:58.232161Z","iopub.execute_input":"2022-01-20T08:52:58.232537Z","iopub.status.idle":"2022-01-20T08:53:01.197599Z","shell.execute_reply.started":"2022-01-20T08:52:58.232461Z","shell.execute_reply":"2022-01-20T08:53:01.196863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype.name\n\n        if col_type not in ['object', 'category', 'datetime64[ns, UTC]', 'datetime64[ns]']:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:01.199476Z","iopub.execute_input":"2022-01-20T08:53:01.199723Z","iopub.status.idle":"2022-01-20T08:53:01.213695Z","shell.execute_reply.started":"2022-01-20T08:53:01.199691Z","shell.execute_reply":"2022-01-20T08:53:01.212865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\n# df_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:01.216358Z","iopub.execute_input":"2022-01-20T08:53:01.216819Z","iopub.status.idle":"2022-01-20T08:53:52.726392Z","shell.execute_reply.started":"2022-01-20T08:53:01.216715Z","shell.execute_reply":"2022-01-20T08:53:52.725642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n# df_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.728537Z","iopub.execute_input":"2022-01-20T08:53:52.728772Z","iopub.status.idle":"2022-01-20T08:53:52.746212Z","shell.execute_reply.started":"2022-01-20T08:53:52.728739Z","shell.execute_reply":"2022-01-20T08:53:52.745578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, \n                 asset_id, \n                 train=True):\n    '''\n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    df = df.sort_values('timestamp')\n    if train == True:\n        df_feat = df.copy()\n        # define a train_flg column to split your data into train and validation\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n        df_feat['train_flg'] = np.where(df_feat['timestamp']>=valid_window[0], 0,1)\n        df_feat = df_feat[['timestamp','Asset_ID', 'Count','Open', 'High', 'Low','Close','Volume', 'Target','train_flg']].copy()\n    else:\n        df = df.sort_values('row_id')\n        df_feat = df[['Asset_ID','Count','Open', 'High', 'Low', 'Close','Volume','row_id']].copy()\n    \n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling(15).mean()/df_feat['Close'] -1\n    df_feat['sma60'] = df_feat['Close'].rolling(60).mean()/df_feat['Close'] -1\n    df_feat['sma240'] = df_feat['Close'].rolling(240).mean()/df_feat['Close'] -1\n    \n    #df_feat['return15'] = df_feat['Close']/df_feat['Close'].shift(15) -1\n    #df_feat['return60'] = df_feat['Close']/df_feat['Close'].shift(60) -1\n    #df_feat['return240'] = df_feat['Close']/df_feat['Close'].shift(240) -1\n    \n    df_feat['upper_shadow'] = df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open'])\n    df_feat['lower_shadow'] = np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low']\n    #df['Mean'] = df[['Open', 'High', 'Low', 'Close']].mean()\n    df_feat['Close/Open'] = df_feat['Close'] / df_feat['Open'] \n    df_feat['hlco_ratio'] = (df_feat['High'] - df_feat['Low'])/(df_feat['Close']-df_feat['Open']+1e-6)\n    df_feat['spread'] = df_feat['High'] - df_feat['Low']\n    df_feat['mean_trade'] = df_feat['Volume']/df_feat['Count']\n    #df['log_price_change'] = np.log(df['Close']/df['Open'])\n    df_feat = df_feat.fillna(0)\n    \n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.747365Z","iopub.execute_input":"2022-01-20T08:53:52.747683Z","iopub.status.idle":"2022-01-20T08:53:52.760777Z","shell.execute_reply.started":"2022-01-20T08:53:52.747647Z","shell.execute_reply":"2022-01-20T08:53:52.760140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\n# features = ['Asset_ID','sma15','sma60','sma240', 'upper_shadow', 'lower_shadow'\n#            , 'Close/Open', 'hlco_ratio', 'spread', 'mean_trade'\n#            ]\nfeatures = ['Asset_ID','Count','Open', 'High', 'Low', 'Close','Volume']\ncategoricals = ['Asset_ID']","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.761741Z","iopub.execute_input":"2022-01-20T08:53:52.762462Z","iopub.status.idle":"2022-01-20T08:53:52.778403Z","shell.execute_reply.started":"2022-01-20T08:53:52.762425Z","shell.execute_reply":"2022-01-20T08:53:52.777766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef weighted_correlation(a, train_data):\n    \n    weights = train_data.add_w.values.flatten()\n    b = train_data.get_label()\n    \n    \n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'eval_wcorr', corr, True","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.779572Z","iopub.execute_input":"2022-01-20T08:53:52.780551Z","iopub.status.idle":"2022-01-20T08:53:52.789690Z","shell.execute_reply.started":"2022-01-20T08:53:52.780511Z","shell.execute_reply":"2022-01-20T08:53:52.788748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import optuna\nfrom lightgbm import LGBMRegressor # But do not call lightgbm! This is a must! This trick is explained above!\nfrom joblib import parallel_backend","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.791115Z","iopub.execute_input":"2022-01-20T08:53:52.791557Z","iopub.status.idle":"2022-01-20T08:53:52.799140Z","shell.execute_reply.started":"2022-01-20T08:53:52.791515Z","shell.execute_reply":"2022-01-20T08:53:52.798323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def objective(trial, asset_id, categoricals, cv_fold_func = np.average):\n    from lightgbm import LGBMRegressor\n    # Optuna suggest params\n    param_lgb = {\n        \"verbosity\": -1,\n        \"boosting_type\": \"gbdt\",\n        \"lambda_l1\": trial.suggest_float(\"lambda_l1\", 1e-8, 10.0, log=True),\n        \"lambda_l2\": trial.suggest_float(\"lambda_l2\", 1e-8, 10.0, log=True),\n        \"num_leaves\": trial.suggest_int(\"num_leaves\", 2, 256),\n        \"feature_fraction\": trial.suggest_float(\"feature_fraction\", 0.4, 1.0),\n        \"bagging_fraction\": trial.suggest_float(\"bagging_fraction\", 0.4, 1.0),\n        \"bagging_freq\": trial.suggest_int(\"bagging_freq\", 1, 7),\n        \"min_child_samples\": trial.suggest_int(\"min_child_samples\", 5, 100),\n        'device': 'gpu',\n    }    \n    # fit for all folds and return composite MAE score\n    \n    feature_df = get_features(df_train, asset_id, train=True)\n    feature_df = pd.merge(feature_df, df_asset_details[['Asset_ID','Weight']], how='left', on=['Asset_ID'])\n    feature_df = reduce_mem_usage(feature_df)\n    for c in categoricals:\n        feature_df[c] = pd.Series(feature_df[c], dtype = 'category')\n    x_train = feature_df.query('train_flg == 1')[features]\n    y_train = feature_df.query('train_flg == 1')['Target'].values\n    x_val = feature_df.query('train_flg == 0')[features]\n    y_val = feature_df.query('train_flg == 0')['Target'].values\n\n    clf = LGBMRegressor(**param_lgb)\n    clf.fit(x_train, y_train)\n    preds = clf.predict(x_val)\n    mae = mean_absolute_error(y_val, preds)\n    \n\n\n    return -1.0 * mae","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.800231Z","iopub.execute_input":"2022-01-20T08:53:52.800437Z","iopub.status.idle":"2022-01-20T08:53:52.814662Z","shell.execute_reply.started":"2022-01-20T08:53:52.800406Z","shell.execute_reply":"2022-01-20T08:53:52.813818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_JOBS = 2\nN_TRIALS = 5","metadata":{"execution":{"iopub.status.busy":"2022-01-20T08:53:52.816092Z","iopub.execute_input":"2022-01-20T08:53:52.816408Z","iopub.status.idle":"2022-01-20T08:53:52.824698Z","shell.execute_reply.started":"2022-01-20T08:53:52.816347Z","shell.execute_reply":"2022-01-20T08:53:52.823968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from optuna.visualization import plot_param_importances\ndef get_best_params(objective, N_TRIALS, N_JOBS, asset_id, categoricals):\n    with parallel_backend('multiprocessing'):\n        study = optuna.create_study(direction = \"maximize\")\n        study.optimize(lambda trial: objective(trial, asset_id, categoricals), n_trials = N_TRIALS, n_jobs = N_JOBS)\n\n    print(\"Number of finished trials: {}\".format(len(study.trials)))\n\n    print(\"Best trial:\")\n    trial = study.best_trial\n\n    print(\"  Value: {}\".format(trial.value))\n\n    print(\"  Params: \")\n    for key, value in trial.params.items():\n        print(\"    {}: {}\".format(key, value))\n    \n\n#     display(optuna.visualization.plot_optimization_history(study))\n#     display(optuna.visualization.plot_slice(study))\n#     display(optuna.visualization.plot_parallel_coordinate(study))\n#     display(optuna.visualization.plot_param_importances (study))\n    best_params = trial.params      \n    return best_params","metadata":{"execution":{"iopub.status.busy":"2022-01-20T09:04:28.554322Z","iopub.execute_input":"2022-01-20T09:04:28.555109Z","iopub.status.idle":"2022-01-20T09:04:28.564425Z","shell.execute_reply.started":"2022-01-20T09:04:28.555064Z","shell.execute_reply":"2022-01-20T09:04:28.563454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_params = {}\nfor asset_id in range(14):\n    best_params[asset_id] = get_best_params(objective, N_TRIALS, N_JOBS, asset_id, categoricals)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T09:04:30.609386Z","iopub.execute_input":"2022-01-20T09:04:30.609640Z","iopub.status.idle":"2022-01-20T09:18:53.258577Z","shell.execute_reply.started":"2022-01-20T09:04:30.609610Z","shell.execute_reply":"2022-01-20T09:18:53.255256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_final_model(asset_id, best_params):\n    feature_df = get_features(df_train, asset_id, train=True)\n    feature_df = pd.merge(feature_df, df_asset_details[['Asset_ID','Weight']], how='left', on=['Asset_ID'])\n    feature_df = reduce_mem_usage(feature_df)\n    # define train and validation weights and datasets\n    weights_train = feature_df.query('train_flg == 1')[['Weight']]\n    weights_test = feature_df.query('train_flg == 0')[['Weight']]\n\n    train_dataset = lgb.Dataset(feature_df.query('train_flg == 1')[features], \n                                feature_df.query('train_flg == 1')['Target'].values, \n                                feature_name = features, \n                                categorical_feature= categoricals)\n    val_dataset = lgb.Dataset(feature_df.query('train_flg == 0')[features], \n                              feature_df.query('train_flg == 0')['Target'].values, \n                              feature_name = features, \n                              categorical_feature= categoricals)\n\n    train_dataset.add_w = weights_train\n    val_dataset.add_w = weights_test\n\n    evals_result = {}\n\n    # train LGBM2\n    model = lgb.train(params = best_params,\n                      train_set = train_dataset, \n                      valid_sets = [val_dataset],\n                      early_stopping_rounds=100,\n                      verbose_eval = 10,\n                      feval=weighted_correlation,\n                      evals_result = evals_result \n                     )\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-17T18:51:52.075855Z","iopub.status.idle":"2022-01-17T18:51:52.076296Z","shell.execute_reply.started":"2022-01-17T18:51:52.076064Z","shell.execute_reply":"2022-01-17T18:51:52.076089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\nfor asset_id in range(14):\n    model = get_final_model(asset_id, best_params[asset_id])\n    models[asset_id] = model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Important!","metadata":{}},{"cell_type":"code","source":"# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nmax_lookback = 250","metadata":{"execution":{"iopub.status.busy":"2022-01-17T18:51:52.07779Z","iopub.status.idle":"2022-01-17T18:51:52.078274Z","shell.execute_reply.started":"2022-01-17T18:51:52.078039Z","shell.execute_reply":"2022-01-17T18:51:52.078065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now we will submit via api\n\n- As mentioned by the host here https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/290412 - the api takes 10 minutes to complete when submitted on the full test data with a simple dummy prediction. \n\n- Therefore, any extra logic we include within the api loop with increase the time to completion significantly.\n\n- I have not focused on optimisation of the logic within this loop yet - there are definetly significant improvements you can try for yourself. For example, using numpy arrays instead of pandas dataframes may help.\n\n- For this version - the submission time is roughly 5 hours.","metadata":{}},{"cell_type":"code","source":"start = time.time()\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\n# create dataframe to store data from the api to create lagged features\nhistory = pd.DataFrame()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    \n    # concatenate new api data to history dataframe\n    history = pd.concat([history, df_test[['timestamp','Asset_ID', 'Count','Open', 'High', 'Low', 'Close', 'Volume', 'row_id']]])\n    for j , row in df_test.iterrows():\n        model = models[row['Asset_ID']]\n        # get features using history dataframe\n        row_features = get_features(history, row['Asset_ID'], train=False)\n        row = row_features.iloc[-1].fillna(0)\n        y_pred = model.predict(row[features])[0]\n\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    \n    # we only want to keep the necessary recent part of our history dataframe, which will depend on your\n    # max_lookback value (your furthest lookback in creating lagged features).\n    history = history.sort_values(by='row_id')\n    history = history.iloc[-(max_lookback*14+100):]\n    \n    # Send submissions\n    env.predict(df_pred)\nstop = time.time()\nprint(stop-start)","metadata":{"execution":{"iopub.status.busy":"2022-01-17T18:51:52.079891Z","iopub.status.idle":"2022-01-17T18:51:52.080446Z","shell.execute_reply.started":"2022-01-17T18:51:52.080201Z","shell.execute_reply":"2022-01-17T18:51:52.080227Z"},"trusted":true},"execution_count":null,"outputs":[]}]}