{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Submitting Lagged Features via API\n\nIn this notebook we submit a lagged features via the API.\n\nThe API works by providing a single row for each Asset - one timestamp at a time - to prevent using future data in predictions.\n\nIn order to utilise lagged features in our model, we must store the outputs from the API so we can calculate features using past data.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:59:24.616713Z","iopub.execute_input":"2021-12-15T05:59:24.617444Z","iopub.status.idle":"2021-12-15T05:59:26.997884Z","shell.execute_reply.started":"2021-12-15T05:59:24.617293Z","shell.execute_reply":"2021-12-15T05:59:26.996786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-15T05:59:26.999953Z","iopub.execute_input":"2021-12-15T05:59:27.000799Z","iopub.status.idle":"2021-12-15T06:00:23.774208Z","shell.execute_reply.started":"2021-12-15T05:59:27.000739Z","shell.execute_reply":"2021-12-15T06:00:23.773524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:23.775395Z","iopub.execute_input":"2021-12-15T06:00:23.775731Z","iopub.status.idle":"2021-12-15T06:00:23.799530Z","shell.execute_reply.started":"2021-12-15T06:00:23.775696Z","shell.execute_reply":"2021-12-15T06:00:23.798587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, \n                 asset_id, \n                 train=True):\n    '''\n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    df = df.sort_values('timestamp')\n    if train == True:\n        df_feat = df.copy()\n        # define a train_flg column to split your data into train and validation\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n        df_feat['train_flg'] = np.where(df_feat['timestamp']>=valid_window[0], 0,1)\n        df_feat = df_feat[['timestamp','Asset_ID','Close','Target','train_flg']].copy()\n    else:\n        df = df.sort_values('row_id')\n        df_feat = df[['Asset_ID','Close','row_id']].copy()\n    \n    # Create your features here, they can be lagged or not\n    df_feat['sma15'] = df_feat['Close'].rolling(15).mean()/df_feat['Close'] -1\n    df_feat['sma60'] = df_feat['Close'].rolling(60).mean()/df_feat['Close'] -1\n    df_feat['sma240'] = df_feat['Close'].rolling(240).mean()/df_feat['Close'] -1\n    \n    '''\n    \n    df_feat['return15'] = df_feat['Close'][:-15]/df_feat['Close'][15:] -1\n    df_feat['return60'] = df_feat['Close'][:-60]/df_feat['Close'][60:] -1\n    df_feat['return240'] = df_feat['Close'][:-240]/df_feat['Close'][240:] -1\n    ''' \n    df_feat = df_feat.fillna(0)\n    \n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:23.801884Z","iopub.execute_input":"2021-12-15T06:00:23.802583Z","iopub.status.idle":"2021-12-15T06:00:23.810836Z","shell.execute_reply.started":"2021-12-15T06:00:23.802529Z","shell.execute_reply":"2021-12-15T06:00:23.810195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create your feature dataframe for each asset and concatenate\nfeature_df = pd.DataFrame()\nfor i in range(14):\n    feature_df = pd.concat([feature_df,get_features(df_train,i,train=True)])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:23.811993Z","iopub.execute_input":"2021-12-15T06:00:23.812260Z","iopub.status.idle":"2021-12-15T06:00:44.804195Z","shell.execute_reply.started":"2021-12-15T06:00:23.812231Z","shell.execute_reply":"2021-12-15T06:00:44.802983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assign weight column feature dataframe\nfeature_df = pd.merge(feature_df, df_asset_details[['Asset_ID','Weight']], how='left', on=['Asset_ID'])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:44.805573Z","iopub.execute_input":"2021-12-15T06:00:44.805828Z","iopub.status.idle":"2021-12-15T06:00:48.478685Z","shell.execute_reply.started":"2021-12-15T06:00:44.805798Z","shell.execute_reply":"2021-12-15T06:00:48.477908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nfeatures = ['Asset_ID','sma15','sma60','sma240','return15','return60','return240']\ncategoricals = ['Asset_ID']","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.480051Z","iopub.execute_input":"2021-12-15T06:00:48.480770Z","iopub.status.idle":"2021-12-15T06:00:48.485024Z","shell.execute_reply.started":"2021-12-15T06:00:48.480735Z","shell.execute_reply":"2021-12-15T06:00:48.484478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef weighted_correlation(a, train_data):\n    \n    weights = train_data.add_w.values.flatten()\n    b = train_data.get_label()\n    \n    \n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'eval_wcorr', corr, True","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.486451Z","iopub.execute_input":"2021-12-15T06:00:48.486976Z","iopub.status.idle":"2021-12-15T06:00:48.497921Z","shell.execute_reply.started":"2021-12-15T06:00:48.486942Z","shell.execute_reply":"2021-12-15T06:00:48.497283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Editing starts here\nSome kernels are saving the entire previous history dataframe for prediction that sounds really bizzare, I mean the memory usage will be way too high . So here rather than saving all the previous samples we only save the computations from the past sample , in this way we can save up a lot of memory , otherwise we may face a lot of compute_issues.\n\n[Note] Just for the sake of an example i am submitting one of the created features as the predictions and not training a new model to do so","metadata":{}},{"cell_type":"markdown","source":"# Functions :-","metadata":{}},{"cell_type":"code","source":"# For the rolling average we can only store samples for the respective window  , say 15 minutes mot the others as that sounds dumb and not needed\n# I will also give some additional functions for you other than rolling average so that you need to code them out\n\n\n\n\nclass RollingAverage():\n    '''\n    This code is for saving up ram in Rolling average  lagged feats\n    '''\n    def __init__(self,windows=[]):\n          \n            self.max_length=max(windows)\n            self.dataframes=[]\n    def compute(self,current):\n        self.dataframes.append(float(current['Close']))\n       \n        if len(self.dataframes)>self.max_length: \n            self.dataframes.pop(0)  # This sample is not needed anymore so we can remove it\n      \n       \n        min15avg=np.mean(np.array(self.dataframes)[max([-1*len(self.dataframes),-15]):]) # 15min window average\n        min60avg=np.mean(np.array(self.dataframes)[max([-1*len(self.dataframes),-65]):]) # 16min window average\n        min240avg=np.mean(np.array(self.dataframes)[max([-1*len(self.dataframes),-240]):])# 240 min window average\n        # Compute features here \n        current['sma15'] = min15avg/current['Close'] - 1 \n        current['sma60'] = min60avg/current['Close'] - 1\n        current['sma240'] = min240avg/current['Close'] - 1\n        \n        \n        return current\n    \n\n        \n        \n        \n        \n        \n        \n        \n        \n        \n        \n      \n        \n        \n                \n        \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.499292Z","iopub.execute_input":"2021-12-15T06:00:48.499536Z","iopub.status.idle":"2021-12-15T06:00:48.509947Z","shell.execute_reply.started":"2021-12-15T06:00:48.499508Z","shell.execute_reply":"2021-12-15T06:00:48.509323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfcrop=df_train[df_train['Asset_ID']==0]\ndfcrop.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.512464Z","iopub.execute_input":"2021-12-15T06:00:48.512982Z","iopub.status.idle":"2021-12-15T06:00:48.825511Z","shell.execute_reply.started":"2021-12-15T06:00:48.512947Z","shell.execute_reply":"2021-12-15T06:00:48.824641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rollingavg=RollingAverage([15,60,240])\n","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.826836Z","iopub.execute_input":"2021-12-15T06:00:48.827249Z","iopub.status.idle":"2021-12-15T06:00:48.832234Z","shell.execute_reply.started":"2021-12-15T06:00:48.827204Z","shell.execute_reply":"2021-12-15T06:00:48.831470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rollingavg.max_length","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.833442Z","iopub.execute_input":"2021-12-15T06:00:48.833723Z","iopub.status.idle":"2021-12-15T06:00:48.846952Z","shell.execute_reply.started":"2021-12-15T06:00:48.833684Z","shell.execute_reply":"2021-12-15T06:00:48.846091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rollingavg.compute(dfcrop.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.848079Z","iopub.execute_input":"2021-12-15T06:00:48.848340Z","iopub.status.idle":"2021-12-15T06:00:48.875011Z","shell.execute_reply.started":"2021-12-15T06:00:48.848312Z","shell.execute_reply":"2021-12-15T06:00:48.874136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rollingavg.compute(dfcrop.iloc[1])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.877311Z","iopub.execute_input":"2021-12-15T06:00:48.878400Z","iopub.status.idle":"2021-12-15T06:00:48.892260Z","shell.execute_reply.started":"2021-12-15T06:00:48.878346Z","shell.execute_reply":"2021-12-15T06:00:48.891147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nrollingavg.compute(dfcrop.iloc[2])","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.894331Z","iopub.execute_input":"2021-12-15T06:00:48.895094Z","iopub.status.idle":"2021-12-15T06:00:48.912301Z","shell.execute_reply.started":"2021-12-15T06:00:48.895003Z","shell.execute_reply":"2021-12-15T06:00:48.911272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avgs=[RollingAverage([15,60,240]) for _ in range(14)] # Create 14 different objects one for each asset","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.913898Z","iopub.execute_input":"2021-12-15T06:00:48.914677Z","iopub.status.idle":"2021-12-15T06:00:48.924745Z","shell.execute_reply.started":"2021-12-15T06:00:48.914631Z","shell.execute_reply":"2021-12-15T06:00:48.923842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Exponentially Weighted Average\nclass ExponentiallyWeightedAverage:\n    v1=0\n    def __init__(self,beta):\n        self.beta=beta\n    def compute(self,value):\n        self.v1=self.beta*value+(1-self.beta)*self.v1\n        return self.v1\newm=ExponentiallyWeightedAverage(0.75)\nprint(ewm.compute(dfcrop.iloc[0]))\nprint(ewm.compute(dfcrop.iloc[1]))       \nprint(ewm.compute(dfcrop.iloc[2]))     ","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.926309Z","iopub.execute_input":"2021-12-15T06:00:48.927678Z","iopub.status.idle":"2021-12-15T06:00:48.947841Z","shell.execute_reply.started":"2021-12-15T06:00:48.927614Z","shell.execute_reply":"2021-12-15T06:00:48.947129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfcrop.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.949096Z","iopub.execute_input":"2021-12-15T06:00:48.949594Z","iopub.status.idle":"2021-12-15T06:00:48.973478Z","shell.execute_reply.started":"2021-12-15T06:00:48.949522Z","shell.execute_reply":"2021-12-15T06:00:48.972076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Measures the rate of change of a feature over time\nclass RateCalculator:\n    def __init__(self,default=0.0):\n        # Default value is rate for first timestamp\n        self.default=default\n        self.previous=False\n    def compute(self,row):\n        if type(self.previous)==type(False):\n            self.previous=row.copy()\n            row[list(row.keys())]=self.default\n            \n            return row\n        \n        final=row/self.previous\n        self.previous=row\n        return final\n    \nrate=RateCalculator()\nprint(rate.compute(dfcrop.iloc[0]))\n\nprint(rate.compute(dfcrop.iloc[1]))       \nprint(rate.compute(dfcrop.iloc[2]))     \n\n        \n        \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.974874Z","iopub.execute_input":"2021-12-15T06:00:48.975309Z","iopub.status.idle":"2021-12-15T06:00:48.990646Z","shell.execute_reply.started":"2021-12-15T06:00:48.975264Z","shell.execute_reply":"2021-12-15T06:00:48.989581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avgs=[RollingAverage([15,60,240]) for _ in range(14)] # Create 14 different objects one for each asset\newms=[ExponentiallyWeightedAverage(0.75) for _ in range(14)]\nrates=[RateCalculator(1.0) for _ in range(14)]\n","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:48.992453Z","iopub.execute_input":"2021-12-15T06:00:48.993011Z","iopub.status.idle":"2021-12-15T06:00:49.005242Z","shell.execute_reply.started":"2021-12-15T06:00:48.992968Z","shell.execute_reply":"2021-12-15T06:00:49.004460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"start = time.time()\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\n# create dataframe to store data from the api to create lagged features\nhistory = pd.DataFrame()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    \n\n    for j , row in df_test.iterrows():\n        # get features using history dataframe\n        avg=avgs[int(row['Asset_ID'])]\n        ewm=ewms[int(row['Asset_ID'])]\n        rate=rates[int(row['Asset_ID'])]\n        row_features1=avg.compute(row)\n        row_features2=ewm.compute(row)\n        row_features3=rate.compute(row)\n      \n        y_pred = float(row_features1['sma15']) # Giving a naive submission for now\n\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    \n    # we only want to keep the necessary recent part of our history dataframe, which will depend on your\n    # max_lookback value (your furthest lookback in creating lagged features).\n \n    \n    # Send submissions\n    env.predict(df_pred)\nstop = time.time()\nprint(stop-start)","metadata":{"execution":{"iopub.status.busy":"2021-12-15T06:00:49.006754Z","iopub.execute_input":"2021-12-15T06:00:49.007007Z","iopub.status.idle":"2021-12-15T06:00:49.244835Z","shell.execute_reply.started":"2021-12-15T06:00:49.006978Z","shell.execute_reply":"2021-12-15T06:00:49.243258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}