{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-24T09:46:21.427261Z","iopub.execute_input":"2021-12-24T09:46:21.427789Z","iopub.status.idle":"2021-12-24T09:46:21.542592Z","shell.execute_reply.started":"2021-12-24T09:46:21.427695Z","shell.execute_reply":"2021-12-24T09:46:21.541880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport gc\nimport traceback\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport gresearch_crypto\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\n#import pandas as pd, numpy as np\nfrom tensorflow.keras import layers\nimport tensorflow_probability as tfp\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import RobustScaler\nfrom tensorflow.keras.preprocessing.sequence import TimeseriesGenerator\npd.set_option('display.max_columns', None)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:46:21.543718Z","iopub.execute_input":"2021-12-24T09:46:21.543934Z","iopub.status.idle":"2021-12-24T09:46:30.895064Z","shell.execute_reply.started":"2021-12-24T09:46:21.543903Z","shell.execute_reply":"2021-12-24T09:46:30.893593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEVICE = \"TPU\" #or \"GPU\"\n\nSEED = 42\n\nEPOCHS = 25\nDEBUG = True\nN_ASSETS = 14\nWINDOW_SIZE = 15\nBATCH_SIZE = 1024\nPCT_VALIDATION = 10 # last 10% of the data are used as validation set","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:46:30.896245Z","iopub.execute_input":"2021-12-24T09:46:30.896584Z","iopub.status.idle":"2021-12-24T09:46:30.902359Z","shell.execute_reply.started":"2021-12-24T09:46:30.896552Z","shell.execute_reply":"2021-12-24T09:46:30.901260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if DEVICE == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.experimental.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except: print(\"failed to initialize TPU\")\n    else: DEVICE = \"GPU\"\n\nif DEVICE != \"TPU\": strategy = tf.distribute.get_strategy()\nif DEVICE == \"GPU\": print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:46:30.904378Z","iopub.execute_input":"2021-12-24T09:46:30.904888Z","iopub.status.idle":"2021-12-24T09:46:30.946826Z","shell.execute_reply.started":"2021-12-24T09:46:30.904853Z","shell.execute_reply":"2021-12-24T09:46:30.945853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datatable as dt\nextra_data_files = {0: '../input/cryptocurrency-extra-data-binance-coin', 2: '../input/cryptocurrency-extra-data-bitcoin-cash', 1: '../input/cryptocurrency-extra-data-bitcoin', 3: '../input/cryptocurrency-extra-data-cardano', 4: '../input/cryptocurrency-extra-data-dogecoin', 5: '../input/cryptocurrency-extra-data-eos-io', 6: '../input/cryptocurrency-extra-data-ethereum', 7: '../input/cryptocurrency-extra-data-ethereum-classic', 8: '../input/cryptocurrency-extra-data-iota', 9: '../input/cryptocurrency-extra-data-litecoin', 11: '../input/cryptocurrency-extra-data-monero', 10: '../input/cryptocurrency-extra-data-maker', 12: '../input/cryptocurrency-extra-data-stellar', 13: '../input/cryptocurrency-extra-data-tron'}\n\n# Uncomment to load the original csv [slower]\n# orig_df_train = pd.read_csv(data_path + 'train.csv') \n# supp_df_train = pd.read_csv(data_path + 'supplemental_train.csv')\n# df_asset_details = pd.read_csv(data_path  + 'asset_details.csv').sort_values(\"Asset_ID\")\n\norig_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_train.jay').to_pandas()\ndf_asset_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nsupp_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas()\nassets_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nasset_weight_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Weight'].tolist()[idx] for idx in range(len(assets_details))}\nasset_name_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Asset_Name'].tolist()[idx] for idx in range(len(assets_details))}\n\ndef load_training_data_for_asset(asset_id, load_jay = True):\n    dfs = []\n    if INCCOMP: dfs.append(orig_df_train[orig_df_train[\"Asset_ID\"] == asset_id].copy())\n    if INCSUPP: dfs.append(supp_df_train[supp_df_train[\"Asset_ID\"] == asset_id].copy())\n    \n    if load_jay:\n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.jay').to_pandas())\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.jay').to_pandas())\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.jay').to_pandas())\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.jay').to_pandas())\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.jay').to_pandas())\n    else: \n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'))\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'))\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'))\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'))\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'))\n    df = pd.concat(dfs, axis = 0) if len(dfs) > 1 else dfs[0]\n    df['date'] = pd.to_datetime(df['timestamp'], unit = 's')\n    if LOAD_STRICT: df = df.loc[df['date'] < \"2021-06-13 00:00:00\"]    \n    df = df.sort_values('date')\n    return df\n\ndef load_data_for_all_assets():\n    dfs = []\n    for asset_id in list(extra_data_files.keys()): dfs.append(load_training_data_for_asset(asset_id))\n    return pd.concat(dfs)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:46:30.948224Z","iopub.execute_input":"2021-12-24T09:46:30.948753Z","iopub.status.idle":"2021-12-24T09:47:00.766798Z","shell.execute_reply.started":"2021-12-24T09:46:30.948716Z","shell.execute_reply":"2021-12-24T09:47:00.765832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD STRICT? YES=1 NO=0 | see: https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm\nLOAD_STRICT = True\n\n# WHICH YEARS TO INCLUDE? YES=1 NO=0\nINC2021 = 0\nINC2020 = 0\nINC2019 = 0\nINC2018 = 0\nINC2017 = 0\nINCCOMP = 1\nINCSUPP = 0\n\ntrain = load_data_for_all_assets().sort_values('timestamp').set_index(\"timestamp\")\nif DEBUG: train = train[10000000:]\n\ntest = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_example_test.jay').to_pandas()\nsample_prediction_df = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_example_sample_submission.jay').to_pandas()\nassets = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nprint(assets)\nassets_order = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas().Asset_ID[:N_ASSETS]\nassets_order = dict((t,i) for i,t in enumerate(assets_order))\nprint(assets_order.keys())\nprint(\"Loaded all data!\")","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:47:00.768108Z","iopub.execute_input":"2021-12-24T09:47:00.768396Z","iopub.status.idle":"2021-12-24T09:47:12.964601Z","shell.execute_reply.started":"2021-12-24T09:47:00.768364Z","shell.execute_reply":"2021-12-24T09:47:12.963364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wght_arr = [assets.set_index('Asset_ID').loc[x ,'Weight']  for x in assets_order.keys() ]\ntot_wght_arr = sum(wght_arr)\nwght_arr","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:47:12.965949Z","iopub.execute_input":"2021-12-24T09:47:12.966293Z","iopub.status.idle":"2021-12-24T09:47:12.992451Z","shell.execute_reply.started":"2021-12-24T09:47:12.966247Z","shell.execute_reply":"2021-12-24T09:47:12.991434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype.name\n        \n        if col_type not in ['object', 'category', 'datetime64[ns, UTC]', 'datetime64[ns]']:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:47:12.994506Z","iopub.execute_input":"2021-12-24T09:47:12.995230Z","iopub.status.idle":"2021-12-24T09:47:13.014437Z","shell.execute_reply.started":"2021-12-24T09:47:12.995175Z","shell.execute_reply":"2021-12-24T09:47:13.012448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\ndef lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef get_features(df, row = False):\n    df_feat = df\n    df_feat['spread'] = df_feat['High'] - df_feat['Low']\n    df_feat['mean_trade'] = df_feat['Volume']/df_feat['Count']\n    df_feat['log_price_change'] = np.log(df_feat['Close']/df_feat['Open'])\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    df_feat['trade'] = df_feat['Close'] - df_feat['Open']\n    df_feat['gtrade'] = df_feat['trade'] / df_feat['Count']\n    df_feat['shadow1'] = df_feat['trade'] / df_feat['Volume']\n    df_feat['shadow3'] = df_feat['upper_Shadow'] / df_feat['Volume']\n    df_feat['shadow5'] = df_feat['lower_Shadow'] / df_feat['Volume']\n    df_feat['diff1'] = df_feat['Volume'] - df_feat['Count']\n    df_feat['mean1'] = (df_feat['shadow5'] + df_feat['shadow3']) / 2\n    df_feat['mean2'] = (df_feat['shadow1'] + df_feat['Volume']) / 2\n    df_feat['mean3'] = (df_feat['trade'] + df_feat['gtrade']) / 2\n    df_feat['mean4'] = (df_feat['diff1'] + df_feat['upper_Shadow']) / 2\n    df_feat['mean5'] = (df_feat['diff1'] + df_feat['lower_Shadow']) / 2\n    df_feat['UPS'] = (df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open']))\n    df_feat['UPS'] = df_feat['UPS']\n    df_feat['LOS'] = (np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low'])\n    df_feat['LOS'] = df_feat['LOS']\n    df_feat['RNG'] = ((df_feat['High'] - df_feat['Low']) / df_feat['VWAP'])\n    df_feat['RNG'] = df_feat['RNG']\n    df_feat['MOV'] = ((df_feat['Close'] - df_feat['Open']) / df_feat['VWAP'])\n    df_feat['MOV'] = df_feat['MOV']\n    df_feat['CLS'] = ((df_feat['Close'] - df_feat['VWAP']) / df_feat['VWAP'])\n    df_feat['CLS'] = df_feat['CLS']\n    df_feat['LOGVOL'] = np.log(1. + df_feat['Volume'])\n    df_feat['LOGVOL'] = df_feat['LOGVOL']\n    df_feat['LOGCNT'] = np.log(1. + df_feat['Count'])\n    df_feat['LOGCNT'] = df_feat['LOGCNT']\n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"]\n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"]\n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"]\n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row: df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else: df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis = 1)\n    df_feat[\"High/Mean\"] = df_feat[\"High\"] / df_feat[\"Mean\"]\n    df_feat[\"Low/Mean\"] = df_feat[\"Low\"] / df_feat[\"Mean\"]\n    df_feat[\"Volume/Count\"] = df_feat[\"Volume\"] / (df_feat[\"Count\"] + 1)\n    mean_price = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    median_price = df_feat[['Open', 'High', 'Low', 'Close']].median(axis=1)\n    df_feat['high2mean'] = df_feat['High'] / mean_price\n    df_feat['low2mean'] = df_feat['Low'] / mean_price\n    df_feat['high2median'] = df_feat['High'] / median_price\n    df_feat['low2median'] = df_feat['Low'] / median_price\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    return df_feat\n\ntrain[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']] = train[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Target']].astype(np.float32)\nprint(train.shape)\ntrain['Target'] = train['Target'].fillna(0)\nVWAP_max = np.max(train[np.isfinite(train.VWAP)].VWAP)\nVWAP_min = np.min(train[np.isfinite(train.VWAP)].VWAP)\ntrain['VWAP'] = np.nan_to_num(train.VWAP, posinf=VWAP_max, neginf=VWAP_min)\ndf = train[['Asset_ID', 'Target']].copy()\ntimes = dict((t,i) for i,t in enumerate(df.index.unique()))\ndf['id'] = df.index.map(times)\ndf['id'] = df['id'].astype(str) + '_' + df['Asset_ID'].astype(str)\nids = df.id.copy()\ndel df\ntrain = get_features(train)\ntrain_features = [i for i in train.columns if i not in ['Target', 'date', 'timestamp', 'Asset_ID', 'groups']]","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:47:13.017359Z","iopub.execute_input":"2021-12-24T09:47:13.018280Z","iopub.status.idle":"2021-12-24T09:47:44.202928Z","shell.execute_reply.started":"2021-12-24T09:47:13.018227Z","shell.execute_reply":"2021-12-24T09:47:44.201871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.sort_index()\nind = train.index.unique()\ndef reindex(df):\n    df = df.reindex(range(ind[0],ind[-1]+60,60),method='nearest')\n    df = df.fillna(method=\"ffill\").fillna(method=\"bfill\")\n    return df\ntrain = train.groupby('Asset_ID').apply(reindex).reset_index(0, drop=True).sort_index()\ngc.collect()\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:47:44.205921Z","iopub.execute_input":"2021-12-24T09:47:44.206699Z","iopub.status.idle":"2021-12-24T09:48:11.784794Z","shell.execute_reply.started":"2021-12-24T09:47:44.206650Z","shell.execute_reply":"2021-12-24T09:48:11.783766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:48:11.785992Z","iopub.execute_input":"2021-12-24T09:48:11.786248Z","iopub.status.idle":"2021-12-24T09:48:11.837121Z","shell.execute_reply.started":"2021-12-24T09:48:11.786212Z","shell.execute_reply":"2021-12-24T09:48:11.836215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Matching records and marking generated rows as 'non-real'\ntrain['group_num'] = train.index.map(times)\ntrain = train.dropna(subset=['group_num'])\ntrain['group_num'] = train['group_num'].astype('int')\ntrain['id'] = train['group_num'].astype(str) + '_' + train['Asset_ID'].astype(str)\ntrain['is_real'] = train.id.isin(ids) * 1\ntrain = train.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:48:11.838270Z","iopub.execute_input":"2021-12-24T09:48:11.838626Z","iopub.status.idle":"2021-12-24T09:48:49.423507Z","shell.execute_reply.started":"2021-12-24T09:48:11.838595Z","shell.execute_reply":"2021-12-24T09:48:49.422613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Features values for 'non-real' rows are set to zeros\nfeatures = train.columns.drop(['Asset_ID','group_num','is_real'])\ntrain.loc[train.is_real == 0, features] = 0.","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:48:49.424750Z","iopub.execute_input":"2021-12-24T09:48:49.424981Z","iopub.status.idle":"2021-12-24T09:49:36.675853Z","shell.execute_reply.started":"2021-12-24T09:48:49.424954Z","shell.execute_reply":"2021-12-24T09:49:36.674814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['asset_order'] = train.Asset_ID.map(assets_order)\ntrain = train.sort_values(by=['group_num', 'asset_order'])\ntrain = reduce_mem_usage(train)\ntrain.head(20)\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:49:36.676975Z","iopub.execute_input":"2021-12-24T09:49:36.677209Z","iopub.status.idle":"2021-12-24T09:50:11.523778Z","shell.execute_reply.started":"2021-12-24T09:49:36.677179Z","shell.execute_reply":"2021-12-24T09:50:11.522969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = train['Target'].to_numpy().reshape(-1, N_ASSETS)\nfeatures = train.columns.drop(['Asset_ID', 'Target', 'group_num', 'is_real', 'date'])\ntrain = train[features]\ntrain = train.values\ntrain = train.reshape(-1, N_ASSETS, train.shape[-1])","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:11.525082Z","iopub.execute_input":"2021-12-24T09:50:11.525454Z","iopub.status.idle":"2021-12-24T09:50:18.320440Z","shell.execute_reply.started":"2021-12-24T09:50:11.525418Z","shell.execute_reply":"2021-12-24T09:50:18.319371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.321812Z","iopub.execute_input":"2021-12-24T09:50:18.322039Z","iopub.status.idle":"2021-12-24T09:50:18.327861Z","shell.execute_reply.started":"2021-12-24T09:50:18.322007Z","shell.execute_reply":"2021-12-24T09:50:18.326946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.329687Z","iopub.execute_input":"2021-12-24T09:50:18.330737Z","iopub.status.idle":"2021-12-24T09:50:18.345830Z","shell.execute_reply.started":"2021-12-24T09:50:18.330682Z","shell.execute_reply":"2021-12-24T09:50:18.344784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class sample_generator(keras.utils.Sequence):\n    def __init__(self, x_set, y_set, batch_size, length):\n        self.x, self.y = x_set, y_set\n        self.batch_size = batch_size\n        self.length = length\n        self.size = len(x_set)\n    def __len__(self): return int(np.ceil(len(self.x) / float(self.batch_size)))\n    def __getitem__(self, idx):\n        batch_x=[]\n        batch_y=[]\n        for i in range(self.batch_size):\n            start_ind = self.batch_size*idx + i\n            end_ind = start_ind + self.length \n            if end_ind <= self.size:\n                batch_x.append(self.x[start_ind : end_ind])\n                batch_y.append(self.y[end_ind -1])\n        return np.array(batch_x), np.array(batch_y)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.346884Z","iopub.execute_input":"2021-12-24T09:50:18.347099Z","iopub.status.idle":"2021-12-24T09:50:18.360088Z","shell.execute_reply.started":"2021-12-24T09:50:18.347074Z","shell.execute_reply":"2021-12-24T09:50:18.358761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test = train[:-len(train)//PCT_VALIDATION], train[-len(train)//PCT_VALIDATION:]\ny_train, y_test = targets[:-len(train)//PCT_VALIDATION], targets[-len(train)//PCT_VALIDATION:]","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.362271Z","iopub.execute_input":"2021-12-24T09:50:18.362590Z","iopub.status.idle":"2021-12-24T09:50:18.393484Z","shell.execute_reply.started":"2021-12-24T09:50:18.362560Z","shell.execute_reply":"2021-12-24T09:50:18.392392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator = sample_generator(X_train, y_train, length = WINDOW_SIZE, batch_size = BATCH_SIZE)\nval_generator = sample_generator(X_test, y_test, length = WINDOW_SIZE, batch_size = BATCH_SIZE)\nprint(f'Sample shape: {train_generator[0][0].shape}')\nprint(f'Target shape: {train_generator[0][1].shape}')","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.394894Z","iopub.execute_input":"2021-12-24T09:50:18.395666Z","iopub.status.idle":"2021-12-24T09:50:18.441221Z","shell.execute_reply.started":"2021-12-24T09:50:18.395629Z","shell.execute_reply":"2021-12-24T09:50:18.440057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_generator[0][0][:,:, 1].shape","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.442975Z","iopub.execute_input":"2021-12-24T09:50:18.443204Z","iopub.status.idle":"2021-12-24T09:50:18.466329Z","shell.execute_reply.started":"2021-12-24T09:50:18.443177Z","shell.execute_reply":"2021-12-24T09:50:18.465555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Custom - Loss Functions**","metadata":{}},{"cell_type":"code","source":"def MaxCorrelation(y_true,y_pred): return -tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\ndef Correlation(y_true,y_pred): return tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None))\n\ndef masked_mse(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_squared_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef MaxCorr_masked_mse(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return -tf.math.abs(tfp.stats.correlation(y_pred,y_true, sample_axis=None, event_axis=None)) + tf.keras.losses.mean_squared_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef masked_mae(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.mean_absolute_error(y_true = y_true_masked, y_pred = y_pred_masked)\n\ndef masked_cosine(y_true, y_pred):\n    mask = tf.math.not_equal(y_true, 0.)\n    y_true_masked = tf.boolean_mask(y_true, mask)\n    y_pred_masked = tf.boolean_mask(y_pred, mask)\n    return tf.keras.losses.cosine_similarity(y_true_masked, y_pred_masked)\n\ndef masked_mse_cosine(y_true ,y_pred):\n    return 0.8*masked_cosine(y_true, y_pred) + 0.2*masked_mse(y_true, y_pred)\n\ndef weighted_masked_cosine(y_true ,y_pred):\n    sum_ = 0\n    for i in range(N_ASSETS):\n        num1 = y_true[: ,i]\n        num2 = y_pred[:,i]\n        mask = tf.math.not_equal(num1, 0.)\n        sum_ += wght_arr[i]*tf.keras.losses.cosine_similarity(tf.boolean_mask(num1, mask), tf.boolean_mask(num2, mask))\n    return sum_    \ndef weighted_masked_mse(y_true ,y_pred):\n    sum_ = 0\n    for i in range(N_ASSETS):\n        num1 = y_true[: ,i]\n        num2 = y_pred[:,i]\n        mask = tf.math.not_equal(num1, 0.)\n        sum_ += wght_arr[i]*tf.keras.losses.mean_squared_error(tf.boolean_mask(num1, mask), tf.boolean_mask(num2, mask))\n    return sum_\n\ndef weighted_masked_mse_cosine(y_true ,y_pred):\n    return 0.7*weighted_masked_cosine(y_true ,y_pred) + 0.3*masked_mse(y_true ,y_pred)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.468286Z","iopub.execute_input":"2021-12-24T09:50:18.468905Z","iopub.status.idle":"2021-12-24T09:50:18.483950Z","shell.execute_reply.started":"2021-12-24T09:50:18.468859Z","shell.execute_reply":"2021-12-24T09:50:18.483253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### for testing\n#def weighted_masked_cosine(y_true ,y_pred):\n#    sum_ = 0\n#    for i in range(N_ASSETS):\n#        num1 = y_true[: ,i]\n#        num2 = y_pred[:,i]\n#        mask = tf.math.not_equal(num1, 0.)\n#       print(num1.shape ,num2.shape ,mask.shape, tf.boolean_mask(num1, mask).shape,tf.boolean_mask(num2, mask).shape)\n#        sum_ += wght_arr[i]*tf.keras.losses.cosine_similarity(tf.boolean_mask(num1, mask), tf.boolean_mask(num2, mask))\n#        print(sum_)    \n#    return sum_/tot_wght_arr\n\n#print(weighted_masked_cosine(tf.random.normal((1024,14)) ,tf.random.normal((1024,14))))","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.485475Z","iopub.execute_input":"2021-12-24T09:50:18.486495Z","iopub.status.idle":"2021-12-24T09:50:18.503796Z","shell.execute_reply.started":"2021-12-24T09:50:18.486414Z","shell.execute_reply":"2021-12-24T09:50:18.502784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def weighted_masked_mse(y_true ,y_pred):\n#    sum_ = 0\n#    for i in range(N_ASSETS):\n#        num1 = y_true[: ,i]\n#        num2 = y_pred[:,i]\n#        mask = tf.math.not_equal(num1, 0.)\n#        print(num1.shape ,num2.shape ,mask.shape, tf.boolean_mask(num1, mask).shape,tf.boolean_mask(num2, mask).shape)\n#        sum_ += wght_arr[i]*tf.keras.losses.mean_squared_error(tf.boolean_mask(num1, mask), tf.boolean_mask(num2, mask))\n#    return 0.003*sum_\n#print(weighted_masked_mse(tf.random.normal((1024,14)) ,tf.random.normal((1024,14))))","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.505724Z","iopub.execute_input":"2021-12-24T09:50:18.506593Z","iopub.status.idle":"2021-12-24T09:50:18.523839Z","shell.execute_reply.started":"2021-12-24T09:50:18.506538Z","shell.execute_reply":"2021-12-24T09:50:18.522822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tf.keras.losses.cosine_similarity(tf.constant([1.,1.,1.,1.,1.]),tf.constant([1.,0.,1.,0.,1.]))","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.525225Z","iopub.execute_input":"2021-12-24T09:50:18.525803Z","iopub.status.idle":"2021-12-24T09:50:18.543114Z","shell.execute_reply.started":"2021-12-24T09:50:18.525759Z","shell.execute_reply":"2021-12-24T09:50:18.541851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Custom Attention model\n\n#basic imports\nfrom keras import initializers, regularizers, constraints\n\n#layer object ------ Taken from https://inspiringpeople.github.io/data%20analysis/lstm_attention/\nclass Temporal_Attention(keras.layers.Layer):\n    \n    def __init__(self ,step_dim, W_regularizer=None, b_regularizer=None,W_constraint=None, b_constraint=None,bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Temporal_Attention, self).__init__(**kwargs)\n        \n        \n    def build(self,input_shape):\n        assert len(input_shape) == 3\n        self.W = self.add_weight(shape =(input_shape[-1] ,) ,initializer=self.init ,name='att_W',regularizer=self.W_regularizer, constraint=self.W_constraint)\n        #print(f'shape of W ---{self.W.shape}')\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight(shape =(input_shape[1],),initializer='zero',name='att_b',regularizer=self.b_regularizer,constraint=self.b_constraint)\n            #print(f'shape of b ---{self.b.shape}')\n        else:\n            self.b = None\n        self.built = True \n        \n        \n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None  \n    \n    \n    def call(self, x, mask=None):\n        #print(f\"Shape of x is {x.shape}\")\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        #print(K.reshape(x, (-1, features_dim)).shape ,K.reshape(self.W, (features_dim, 1)).shape)\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        #print(f\"Shape of eij is {eij.shape}\")    \n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        #print(f\"Shape of a is {a.shape}\")  \n        weighted_input = x * a\n        #print(f\"Shape of weighted_input is {weighted_input.shape}\") \n        num = K.sum(weighted_input, axis=1)\n        #print(f\"Shape of num is {num.shape}\") \n        return num\n    \n    \n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n            \n        \n        ","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.544819Z","iopub.execute_input":"2021-12-24T09:50:18.545060Z","iopub.status.idle":"2021-12-24T09:50:18.564834Z","shell.execute_reply.started":"2021-12-24T09:50:18.545031Z","shell.execute_reply":"2021-12-24T09:50:18.563883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l = Temporal_Attention(WINDOW_SIZE)\nx = tf.random.normal((1024,15,32))\nprint(f\"tensor shape -- {x.shape}\")\ny = l(x)\nprint(f\"final tensor shape -- {y.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.566120Z","iopub.execute_input":"2021-12-24T09:50:18.566479Z","iopub.status.idle":"2021-12-24T09:50:18.798783Z","shell.execute_reply.started":"2021-12-24T09:50:18.566443Z","shell.execute_reply":"2021-12-24T09:50:18.798065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_squence_model(x):\n    y = layers.LSTM(units=32, return_sequences=True ,return_state =True)(x)\n    return y[:-1]\n\ndef get_model(n_assets = 14):\n    x_input = keras.Input(shape=(train_generator[0][0].shape[1], n_assets, train_generator[0][0].shape[-1]))\n    branch_outputs = []\n    for i in range(n_assets):\n        a = layers.Lambda(lambda x: x[:,:, i])(x_input) # Slicing the ith asset:\n        a = layers.Masking(mask_value = 0., )(a)\n        a = layers.Dense(32 , activation = tf.keras.layers.LeakyReLU(alpha=0.3))(a)\n        #a = layers.Dropout(0.3)(a)\n        b = get_squence_model(a)\n        c = Temporal_Attention(WINDOW_SIZE)(b[0])\n        a = layers.Concatenate()([c ,b[1]])\n        a = layers.Dropout(0.2)(a)\n        a = layers.Dense(32 ,activation = tf.keras.layers.LeakyReLU(alpha=0.3))(a)\n        #a = layers.GlobalAvgPool1D()(a)\n        branch_outputs.append(a)\n    x = layers.Concatenate()(branch_outputs)\n    #x = layers.Dropout(0.3)(x)\n    x = layers.Dense(units = 128 , activation = tf.keras.layers.LeakyReLU(alpha=0.3))(x)\n    out = layers.Dense(units = n_assets , activation = tf.keras.activations.tanh)(x)\n    model = keras.Model(inputs=x_input, outputs=out)\n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3), loss = weighted_masked_mse_cosine, metrics=[Correlation ,weighted_masked_cosine,masked_mse])\n    return model\n    \nmodel = get_model()\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:18.801444Z","iopub.execute_input":"2021-12-24T09:50:18.802126Z","iopub.status.idle":"2021-12-24T09:50:30.858082Z","shell.execute_reply.started":"2021-12-24T09:50:18.802092Z","shell.execute_reply":"2021-12-24T09:50:30.857034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.keras.utils.plot_model(get_model(n_assets=3), show_shapes=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:30.859688Z","iopub.execute_input":"2021-12-24T09:50:30.860001Z","iopub.status.idle":"2021-12-24T09:50:34.677438Z","shell.execute_reply.started":"2021-12-24T09:50:30.859952Z","shell.execute_reply":"2021-12-24T09:50:34.676655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(features)\n\ntf.random.set_seed(0)\nestop = keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 7, verbose = 0, mode = 'min',restore_best_weights = True)\nscheduler = keras.optimizers.schedules.ExponentialDecay(1e-3, (0.5 * len(X_train) / BATCH_SIZE), 1e-3)\nlr = keras.callbacks.LearningRateScheduler(scheduler, verbose = 1)\nhistory = model.fit(train_generator, validation_data = (val_generator), epochs = EPOCHS, callbacks = [lr, estop])\n\nfig, ax = plt.subplots(1, 2, figsize=(16, 8))\nhistories = pd.DataFrame(history.history)\nepochs = list(range(1,len(histories)+1))\nloss = histories['loss']\nval_loss = histories['val_loss']\nCorrelation = histories['Correlation']\nval_Correlation = histories['val_Correlation']\nax[0].plot(epochs, loss, label = 'Train Loss')\nax[0].plot(epochs, val_loss, label = 'Val Loss')\nax[0].set_title('Losses')\nax[0].set_xlabel('Epoch')\nax[0].legend(loc='upper right')\nax[1].plot(epochs, Correlation, label = 'Train Correlation')\nax[1].plot(epochs, val_Correlation, label = 'Val Correlation')\nax[1].set_title('Correlations')\nax[1].set_xlabel('Epoch')\nax[1].legend(loc='upper right')\nfig.show()\ngc.collect()\n\n# The correlation coefficients by asset for the validation data\npredictions = model.predict(val_generator)\n\nprint('Asset:    Corr. coef.')\nprint('---------------------')\nfor i in range(N_ASSETS):\n    # drop first 14 values in the y_test, since they are absent in val_generator labels\n    y_true = np.squeeze(y_test[WINDOW_SIZE - 1:, i])\n    y_pred = np.squeeze(predictions[:, i])\n    real_target_ind = np.argwhere(y_true!=0)\n    asset_id = list(assets_order.keys())[i]\n    asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\n    print(f\"{asset_name}: {np.corrcoef(y_pred[real_target_ind].flatten(), y_true[real_target_ind].flatten())[0,1]:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-24T09:50:34.678670Z","iopub.execute_input":"2021-12-24T09:50:34.678881Z","iopub.status.idle":"2021-12-24T13:59:08.581092Z","shell.execute_reply.started":"2021-12-24T09:50:34.678856Z","shell.execute_reply":"2021-12-24T13:59:08.580270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(N_ASSETS):\n    # drop first 14 values in the y_test, since they are absent in val_generator labels\n    y_true = np.squeeze(y_test[WINDOW_SIZE - 1:, i])\n    y_pred = np.squeeze(predictions[:, i])\n    print((min(y_true) ,max(y_true)) ,(min(y_pred) ,max(y_pred)) )\n    print(y_true ,y_pred)\n    real_target_ind = np.argwhere(y_true!=0)\n    asset_id = list(assets_order.keys())[i]\n    asset_name = assets[assets.Asset_ID == asset_id]['Asset_Name'].item()\n    print(f\"{asset_name}: {np.corrcoef(y_pred[real_target_ind].flatten(), y_true[real_target_ind].flatten())[0,1]:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:59:08.582718Z","iopub.execute_input":"2021-12-24T13:59:08.583360Z","iopub.status.idle":"2021-12-24T13:59:09.326751Z","shell.execute_reply.started":"2021-12-24T13:59:08.583297Z","shell.execute_reply":"2021-12-24T13:59:09.325584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sup = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas()[:WINDOW_SIZE * (N_ASSETS)]\nplaceholder = get_features(sup)\nplaceholder['asset_order'] = placeholder.Asset_ID.map(assets_order)\ntest_sample = np.array(placeholder[features])\ntest_sample = test_sample.reshape(-1, (N_ASSETS), test_sample.shape[-1])\ntest_sample = np.expand_dims(test_sample, axis=0)\nexample = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_example_test.jay').to_pandas()[:WINDOW_SIZE - 1]\nexample['asset_order'] = example.Asset_ID.map(assets_order) \nexample = example[['Asset_ID','asset_order']]","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:59:09.328482Z","iopub.execute_input":"2021-12-24T13:59:09.328830Z","iopub.status.idle":"2021-12-24T13:59:10.032035Z","shell.execute_reply.started":"2021-12-24T13:59:09.328785Z","shell.execute_reply":"2021-12-24T13:59:10.030979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df = get_features(test_df)\n    test_data = test_df.merge(example, how='outer', on='Asset_ID').sort_values('asset_order')\n    test = np.array(test_data[features].fillna(0))\n    test = test.reshape(-1, 1, N_ASSETS, test.shape[-1])\n    test_sample = np.hstack([test_sample, test])[:,-1 * WINDOW_SIZE:]\n    y_pred = model.predict(test_sample).squeeze().reshape(-1, 1).squeeze()\n    test_data['Target'] = y_pred\n    for _, row in test_df.iterrows():\n        try: sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = test_data.loc[test_data['row_id'] == row['row_id'], 'Target'].item()\n        except: sample_prediction_df.loc[sample_prediction_df['row_id'] == row['row_id'], 'Target'] = 0\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-12-24T13:59:10.033443Z","iopub.execute_input":"2021-12-24T13:59:10.033689Z","iopub.status.idle":"2021-12-24T13:59:28.623621Z","shell.execute_reply.started":"2021-12-24T13:59:10.033659Z","shell.execute_reply":"2021-12-24T13:59:28.622683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}