{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"}],"dockerImageVersionId":30152,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Features to increase your score\n\nIn this notebook I am sharing features which increased scoring of the model for me.\n\nAt the same time I am also sharing all the features which I was considering before condensing it down to only 8 features.\n\n\n\n\nThis notebook is based on Tom Forbes' notebook \"GResearch - Submitting Lagged Features via API\"\nhttps://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV).astype(np.float32)\ndf_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Feature engineering part\n# calculate z-score\ndef zscore(x, window):\n    r = x.rolling(window=window, min_periods = 1)\n    m = r.mean()\n    s = r.std(ddof=0)\n    z = (x-m)/s\n    return z\n\n# calculate different KPI\ndef upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef upper_shadow_15(df): return df['High'].rolling(window=15, min_periods=1).max() - np.maximum(df['Close'], df['Open']).shift(15)\ndef lower_shadow_15(df): return np.minimum(df['Close'], df['Open']).shift(15) - df['Low'].rolling(window=15, min_periods=1).min()\n\n                                                     \ndef upper_shadow_percent(df): return (df['High'] / np.maximum(df['Close'], df['Open'])) -1\ndef lower_shadow_percent(df): return (np.minimum(df['Close'], df['Open']) / df['Low']) -1\n                                                     \ndef upper_shadow_15_perc(df): return (df['High'].rolling(window=15, min_periods=1).max() / np.maximum(df['Close'], df['Open']).shift(15)) -1\ndef lower_shadow_15_perc(df): return (np.minimum(df['Close'], df['Open']).shift(15) / df['Low'].rolling(window=15, min_periods=1).min()) -1\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, \n                 asset_id, \n                 train=True):\n    '''\n    This function takes a dataframe with all asset data and return the lagged features for a single asset.\n    \n    df - Full dataframe with all assets included\n    asset_id - integer from 0-13 inclusive to represent a cryptocurrency asset\n    train - True - you are training your model\n          - False - you are submitting your model via api\n    '''\n    \n    df = df[df['Asset_ID']==asset_id]\n    df = df.sort_values('timestamp')\n    if train == True:\n        df_feat = df.copy()\n        # define a train_flg column to split your data into train and validation\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n        df_feat['train_flg'] = np.where(df_feat['timestamp']>=valid_window[0], 0,1)\n        df_feat = df_feat[['timestamp','Asset_ID','Close','Count', 'Open', 'High', 'Low','Volume','Target','train_flg']].copy()\n    else:\n        df = df.sort_values('row_id')\n        df_feat = df[['Asset_ID','Close','Count', 'Open', 'High', 'Low','Volume','row_id']].copy()\n    \n    # Create your features here\n    \n    df_feat['Z-score_15_Close'] = zscore(df_feat['Close'], 15)\n    df_feat['lower_shadow_15_perc'] = zscore(lower_shadow_15(df_feat), 1440*30*12)\n    df_feat['upper_shadow_15_perc'] = zscore(upper_shadow_15(df_feat), 1440*30*12)\n    df_feat['Return15_%'] = zscore((df_feat['Close'] / df_feat['Close'].shift(15)) - 1, 1440*30*12)\n    df_feat['Return60_%'] = zscore((df_feat['Close'] / df_feat['Close'].shift(60)) - 1, 1440*30*12)\n    df_feat['Candle_body_%'] = zscore((df_feat['Close'] / df_feat['Open']) - 1, 1440*30*12)\n    df_feat['ATR_15_%'] = zscore(((df_feat['High'].rolling(window=15, min_periods=1).max())/ (df_feat['Low'].rolling(window=15, min_periods=1).min()) - 1), 1440*30*12)\n    df_feat['Z-score_return15_60_%'] = zscore(df_feat['Return15_%'], 60)\n\n    df_feat = df_feat.fillna(0)\n    df_feat = df_feat.replace([np.inf, -np.inf], value=0)\n    \n    '''\n    All features created by me or found in other people's notebooks\n    \n    \n    df_feat['Volume / Count'] = df_feat['Volume'] / df_feat['Count'] \n    #df_feat['Close - VWAP'] = df_feat['Close'] - df_feat['VWAP'] \n    df_feat['(Close / VWAP_15) - 1'] = (df_feat['Close'] / df_feat['VWAP'].rolling(window=15, min_periods=1).mean()) - 1\n    df_feat['Close - VWAP_15'] = (df_feat['Close'] - df_feat['VWAP'].rolling(window=15, min_periods=1).mean())\n    df_feat['sma15'] = df_feat['Close'] / df_feat['Close'].rolling(15).mean() -1\n    df_feat['sma60'] = df_feat['Close'] / df_feat['Close'].rolling(60).mean() -1\n    df_feat['sma240'] = df_feat['Close'] / df_feat['Close'].rolling(240).mean() -1\n    \n    df_feat['Bar_Range'] = df_feat['High'] - df_feat['Low']  \n    df_feat['Bar_Range_%'] = (df_feat['High'] / df_feat['Low']) -1\n    \n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_shadow_percent'] = lower_shadow_percent(df_feat)\n    df_feat['upper_Shadow_percent'] = upper_shadow_percent(df_feat)\n    df_feat['lower_shadow_15'] = lower_shadow_15(df_feat)\n    df_feat['upper_shadow_15'] = upper_shadow_15(df_feat)\n    df_feat['lower_shadow_15_perc'] = lower_shadow_15_perc(df_feat)\n    df_feat['upper_shadow_15_perc'] = upper_shadow_15_perc(df_feat)\n    \n    df_feat['Return1'] = df_feat['Close'] - df_feat['Close'].shift(1) \n    df_feat['Return1_%'] = (df_feat['Close'] / df_feat['Close'].shift(1)) - 1\n    df_feat['Return15'] = df_feat['Close'] - df_feat['Close'].shift(15)\n    df_feat['Return15_%'] = (df_feat['Close'] / df_feat['Close'].shift(15)) - 1\n    df_feat['Return60_%'] = (df_feat['Close'] / df_feat['Close'].shift(60)) - 1\n    df_feat['Return240_%'] = (df_feat['Close'] / df_feat['Close'].shift(240)) - 1\n    df_feat['Candle_body'] = df_feat['Close'] - df_feat['Open'] \n    df_feat['Candle_body_%'] = (df_feat['Close'] / df_feat['Open']) - 1\n    df_feat['Candle_body15'] = df_feat['Close'] - df_feat['Open'].shift(14)\n    df_feat['Candle_body15_%'] = (df_feat['Close'] / df_feat['Open'].shift(14)) -1\n    \n    df_feat['Z-score_15_Close'] = zscore(df_feat['Close'], 15)\n    df_feat['Z-score_60_Volume'] = zscore(df_feat['Volume'], 60)\n    \n    df_feat['Mean_15_Close'] = df_feat['Close'].rolling(window=15, min_periods=1).mean()\n    df_feat['Mean_60_Volume'] = df_feat['Volume'].rolling(window=60, min_periods=1).mean()\n    df_feat['Mean_60_Count'] = df_feat['Count'].rolling(window=60, min_periods=1).mean()\n    df_feat['Z-score_60_Count'] = zscore(df_feat['Count'], 60)\n    df_feat['ATR_15'] = (df_feat['High'].rolling(window=15, min_periods=1).max() - df_feat['Low'].rolling(window=15, min_periods=1).min())\n    df_feat['ATR_15_%'] = ((df_feat['High'].rolling(window=15, min_periods=1).max())/ (df_feat['Low'].rolling(window=15, min_periods=1).min()) - 1)\n    \n    df_feat['Z-score_return15_15'] = zscore(df_feat['Return15'], 15)\n    df_feat['Z-score_return15_15_%'] = zscore(df_feat['Return15_%'], 15)\n    df_feat['Z-score_return15_60'] = zscore(df_feat['Return15'], 60)\n    df_feat['Z-score_return15_60_%'] = zscore(df_feat['Return15_%'], 60)\n    df_feat['Z-score_return15_1440'] = zscore(df_feat['Return15'], 1440)\n    df_feat['Z-score_return15_1440_%'] = zscore(df_feat['Return15_%'], 1440)\n    df_feat['Z-score_return15_month'] = zscore(df_feat['Return15'], 1440*30)\n    df_feat['Z-score_return15_month_%'] = zscore(df_feat['Return15_%'], 1440*30)\n    \n    df_feat['VWAP15_median'] = df_feat['VWAP'].rolling(window=15, min_periods=1).median()\n    df_feat['VWAP60_median'] = df_feat['VWAP'].rolling(window=60, min_periods=1).median()\n    df_feat['VWAP240_median'] = df_feat['VWAP'].rolling(window=240, min_periods=1).median()\n    df_feat['VWAPday_median'] = df_feat['VWAP'].rolling(window=1440, min_periods=1).median()\n    \n    df_feat['Median_return15_15'] = df_feat['Return15'].rolling(window=15, min_periods=1).median()\n    df_feat['Median_return15_15_%'] = df_feat['Return15_%'].rolling(window=15, min_periods=1).median()\n    df_feat['Median_return15_60'] = df_feat['Return15'].rolling(window=60, min_periods=1).median()\n    df_feat['Median_return15_60_%'] = df_feat['Return15_%'].rolling(window=60, min_periods=1).median()\n    df_feat['Median_return15_1440'] = df_feat['Return15'].rolling(window=1440, min_periods=1).median()\n    df_feat['Median_return15_1440_%'] = df_feat['Return15_%'].rolling(window=1440, min_periods=1).median()\n    df_feat['Median_return15_month'] = df_feat['Return15'].rolling(window=1440*30, min_periods=1).median()\n    df_feat['Median_return15_month_%'] = df_feat['Return15_%'].rolling(window=1440*30, min_periods=1).median()\n    df_feat['Median_return15_year'] = df_feat['Return15'].rolling(window=1440*30*12, min_periods=1).median()\n    df_feat['Median_return15_year_%'] = df_feat['Return15_%'].rolling(window=1440*30*12, min_periods=1).median()\n    df_feat['Median_return15_3years_%'] = df_feat['Return15_%'].rolling(window=1440*30*12*3, min_periods=1).median()\n    #df_feat['ROC'] =  ((df_feat['Close'] - df_feat['Close'].shift(15)) / df_feat['Close'].shift(15))*100\n    #df_feat['MFI'] \n    \n    #Seasonality features\n    df_feat[\"hour\"] = df_feat[\"datetime\"].dt.hour\n    df_feat[\"day of week\"] = df_feat[\"datetime\"].dt.dayofweek \n    df_feat[\"day\"] = df_feat[\"datetime\"].dt.day\n    \n    \n    '''\n    \n    df_feat = df_feat.drop(columns=['Close','Count', 'Open', 'High', 'Low','Volume'])\n    df_feat = df_feat.astype(np.float32)\n    \n    return df_feat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create your feature dataframe for each asset and concatenate\nfeature_df = pd.DataFrame()\nfor i in range(14):\n    feature_df = pd.concat([feature_df,get_features(df_train,i,train=True)])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assign weight column feature dataframe\nfeature_df = pd.merge(feature_df, df_asset_details[['Asset_ID','Weight']], how='left', on=['Asset_ID'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = feature_df[['Return15_%','Return60_%','Z-score_15_Close', 'ATR_15_%', 'lower_shadow_15','upper_shadow_15',\n          'Candle_body_%', 'Z-score_return15_60_%',\n          'Target'    \n    ]].corr()\ncorr.style.background_gradient(cmap='coolwarm').set_precision(5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define features for LGBM\nfeatures = ['Asset_ID','Return15_%','Return60_%', 'Z-score_15_Close','ATR_15_%', 'Z-score_return15_60_%',\n            'Candle_body_%','lower_shadow_15_perc','upper_shadow_15_perc'\n            ]\ncategoricals = ['Asset_ID']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define the evaluation metric\ndef weighted_correlation(a, train_data):\n    \n    weights = train_data.add_w.values.flatten()\n    b = train_data.get_label()\n    \n    \n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return 'eval_wcorr', corr, True","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define train and validation weights and datasets\n#\n#feature_df = reduce_memory_usage(feature_df)\n#\nweights_train = feature_df.query('train_flg == 1')[['Weight']]\nweights_test = feature_df.query('train_flg == 0')[['Weight']]\n\ntrain_dataset = lgb.Dataset(feature_df.query('train_flg == 1')[features], \n                            feature_df.query('train_flg == 1')['Target'].values, \n                            feature_name = features, \n                            categorical_feature= categoricals)\nval_dataset = lgb.Dataset(feature_df.query('train_flg == 0')[features], \n                          feature_df.query('train_flg == 0')['Target'].values, \n                          feature_name = features, \n                          categorical_feature= categoricals)\n\ntrain_dataset.add_w = weights_train\nval_dataset.add_w = weights_test\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evals_result = {}\nparams = {'n_estimators': 1500,\n        'objective': 'regression',\n        'metric': 'None',\n        'boosting_type': 'gbdt',\n        'max_depth': -1, \n        'learning_rate': 0.01,\n        'seed': 46,\n        'verbose': -1,\n        }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train LGBM2\nmodel = lgb.train(params = params,\n                  train_set = train_dataset, \n                  valid_sets = [val_dataset],\n                  early_stopping_rounds=100,\n                  verbose_eval = 10,\n                  feval=weighted_correlation,\n                  evals_result = evals_result \n                 )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Important!","metadata":{}},{"cell_type":"code","source":"# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nmax_lookback = 1440*31*12","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Now we will submit via api\n\n- As mentioned by the host here https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/290412 - the api takes 10 minutes to complete when submitted on the full test data with a simple dummy prediction. \n\n- Therefore, any extra logic we include within the api loop with increase the time to completion significantly.\n\n- I have not focused on optimisation of the logic within this loop yet - there are definetly significant improvements you can try for yourself. For example, using numpy arrays instead of pandas dataframes may help.\n\n- For this version - the submission time is roughly 5 hours.","metadata":{}},{"cell_type":"code","source":"'''\nstart = time.time()\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\n# create dataframe to store data from the api to create lagged features\nhistory = pd.DataFrame()\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    \n    # concatenate new api data to history dataframe\n    history = pd.concat([history, df_test[['timestamp','Asset_ID','Close','Count', 'Open', 'High', 'Low', 'Volume','row_id']]])\n    for j , row in df_test.iterrows():\n        # get features using history dataframe\n        row_features = get_features(history, row['Asset_ID'], train=False)\n        row = row_features.iloc[-1].fillna(0)\n        y_pred = model.predict(row[features])[0]\n\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n    \n    # we only want to keep the necessary recent part of our history dataframe, which will depend on your\n    # max_lookback value (your furthest lookback in creating lagged features).\n    history = history.sort_values(by='row_id')\n    history = history.iloc[-(max_lookback*14+100):]\n    \n    # Send submissions\n    env.predict(df_pred)\nstop = time.time()\nprint(stop-start)\n'''","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}