{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport time\nimport json\nimport sys\nimport os\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-14T06:01:28.089927Z","iopub.execute_input":"2021-11-14T06:01:28.090317Z","iopub.status.idle":"2021-11-14T06:01:28.198126Z","shell.execute_reply.started":"2021-11-14T06:01:28.090209Z","shell.execute_reply":"2021-11-14T06:01:28.197172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nresolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu='')\ntf.config.experimental_connect_to_cluster(resolver)\ntf.tpu.experimental.initialize_tpu_system(resolver)\nstrategy = tf.distribute.TPUStrategy(resolver)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:28.199779Z","iopub.execute_input":"2021-11-14T06:01:28.200015Z","iopub.status.idle":"2021-11-14T06:01:39.954625Z","shell.execute_reply.started":"2021-11-14T06:01:28.199990Z","shell.execute_reply":"2021-11-14T06:01:39.953968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes = {'Asset_ID':'int8', 'row_id':'int32', 'Count':'int32', 'Open':'float64', 'High':'float64',\n          'Low':'float64', 'Close':'float64', 'Volume':'float64', 'VWAP':'float16',\n          \"Upper_Shadow\":'float64', \"Lower_Shadow\":'float64', \"Gap\":'float64'}","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:39.956051Z","iopub.execute_input":"2021-11-14T06:01:39.956300Z","iopub.status.idle":"2021-11-14T06:01:39.961592Z","shell.execute_reply.started":"2021-11-14T06:01:39.956274Z","shell.execute_reply":"2021-11-14T06:01:39.960520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"WINDOW = 3\nBATCH_SIZE = 41536\nEPOCHS = 50\n\nASSET_DETAILS = pd.read_csv(\n    \"../input/g-research-crypto-forecasting/asset_details.csv\").sort_values(by=['Asset_ID'])\n#Collect weights for each asset to replace VWAP\nWEIGHTS = ASSET_DETAILS.Weight.values.tolist()\n#List of Train Columns\nTRAIN_COLS = [\"Count\", \"Open\", \"High\", \"Low\", \"Close\", \"Volume\",\n              \"Weight\", \"Upper_Shadow\", \"Lower_Shadow\", \"Gap\"]\n\ndef reindexer(df):\n    df = df.sort_values(by=['timestamp']).set_index('timestamp')\n    df = df.reindex(range(df.index[0], df.index[-1]+60, 60))\n    return df.interpolate(method='linear')\n\ndef feature_engineering(df):\n    #Add new features\n    df['Weight'] = df.Asset_ID.apply(lambda x: WEIGHTS[int(x)])\n    #Note: If adding new features, remember to add it to TRAIN_COLS\n    df['Upper_Shadow'] = df['High'] - np.maximum(df['Open'], df['Close'])\n    df['Lower_Shadow'] = np.minimum(df['Open'], df['Close']) - df['Low']\n    df[\"Gap\"] = np.abs(df['Close'] - df['Open'])\n    \n    df = df.groupby('Asset_ID').apply(lambda x: reindexer(x)).reset_index(0, drop=True)\n    df = df.sort_values(by='row_id')\n    \n    return df\n\ndef rescale_values(df):\n    asset_id = str(int(df.Asset_ID.values[0]))\n    #print(f\"{asset_id:<2}  \", end='')#, type(asset_id), SCALER.get(asset_id, 0))\n    for col in TRAIN_COLS:\n        if col=='Weight': continue\n        div = SCALER[asset_id][col]['max'] - SCALER[asset_id][col]['min']\n        df[col] = df[col].apply(lambda x: (x - SCALER[asset_id][col]['min'])/div)\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:39.962924Z","iopub.execute_input":"2021-11-14T06:01:39.963349Z","iopub.status.idle":"2021-11-14T06:01:40.002394Z","shell.execute_reply.started":"2021-11-14T06:01:39.963321Z","shell.execute_reply":"2021-11-14T06:01:40.001582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/data-cleaning/scale_saved_values.json\", 'r') as f:\n    SCALER = json.load(f)","metadata":{"_kg_hide-output":true,"scrolled":true,"execution":{"iopub.status.busy":"2021-11-14T06:01:40.004630Z","iopub.execute_input":"2021-11-14T06:01:40.004901Z","iopub.status.idle":"2021-11-14T06:01:40.018517Z","shell.execute_reply.started":"2021-11-14T06:01:40.004873Z","shell.execute_reply":"2021-11-14T06:01:40.017797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeSeriesLoader():\n    '''Creates a windowed dataset for Time Series Model.'''\n    \n    def __init__(self, df, window=3, train=True):\n        '''\n        df         => DataFrame where data will be extracted\n        window     => Length of window for Time Series\n        '''\n        self.df = df\n        self.window = window\n        self.is_train = train\n        self.is_file = True if type(self.df)==str else False\n        #If train, remove some of the tail data to avoid overlapping with test\n        #Remove starting from the min timestamp of test\n        if self.is_train and self.is_file:\n            self.df = self.df[self.df.index < 1623542400]\n        \n        self.X = np.array([])\n        self.ID = np.array([])\n        self.y = np.array([])\n        \n        #Partition data into 4 slices to save on RAM\n        m = 10 #number of slices\n        if self.is_file:\n            n = 2600000 #10% is 25939823 - from previous notebook\n        else:\n            n = int(np.ceil(self.df.shape[0]/m))\n            \n        for i in range(m):\n            self.create_windows(i*n, (i+1)*n) #Initialize data with windows\n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx):\n        '''Fetch data at index idx extending |window| steps back. If train, fetch the targets.'''\n        if self.is_train:\n            return self.X[idx], self.ID[idx], self.y[idx]\n        else:\n            return self.X[idx], self.ID[idx]\n    \n    def create_windows(self, start, end):\n        # Make sure that the data shape is of this pattern\n        needed_shape = (-1, self.window, len(TRAIN_COLS))\n            \n        pardir=\"./last_windows/\"\n        try:\n            os.mkdir(pardir)\n        except:\n            pass\n        \n        self.X_i = []\n        self.ID_i = np.array([])\n        self.y_i = np.array([])\n        self.row = np.array([])\n        \n        #Check self.df is a if file or DataFrame\n        if self.is_train and self.is_file:\n            inter_df = pd.read_csv(self.df, skiprows=range(1, start), nrows=end-start)\n            #print(\"Upper\", inter_df.shape)\n            try:\n                inter_df = inter_df[inter_df.timestamp < 1623542400]\n            except:\n                inter_df = inter_df[inter_df.index < 1623542400]\n        else:\n            inter_df = self.df.iloc[start:end].copy()\n            #print(\"Lower\", inter_df.shape)\n            \n        #Loop through each asset_id\n        for asset_id in set(inter_df.Asset_ID):\n            df = inter_df[inter_df.Asset_ID == asset_id].copy()\n            df = rescale_values(df)\n            \n            windows = [list(range(i, i+self.window)) for i in range(df.shape[0]-self.window+1)]\n            X = []\n            ID = df.Asset_ID.values\n            row = df.row_id.values\n            try:\n                y = df['Target'].values\n            except:\n                y = None\n            \n            #Fetch the last saved window. Otherwise, initialize with zeros.\n            try:\n                temp = np.load(pardir+f\"last_window_id{asset_id}_{self.window}.npz\")['x']\n            except:\n                try:\n                    prefix = \"../input/window-creation/\"\n                    temp = np.load(prefix+f\"last_window_id{asset_id}_{self.window}.npz\")['x']\n                except:\n                    temp = np.zeros((self.window, len(TRAIN_COLS)))\n\n            for i in range(min(df.shape[0], (self.window-1))):\n                temp[:self.window-1] = temp[1:]\n                temp[-1] = df[TRAIN_COLS].iloc[i].values\n                X.append(temp.copy())\n                \n            #Save X\n            self.X_i = np.concatenate((\n                np.array(self.X_i).reshape(needed_shape), #Previous items\n                np.array(X).reshape(needed_shape), #Below windows\n                df[TRAIN_COLS].values[windows].reshape(needed_shape)), #Windows\n                axis=0\n            )\n            #save ID\n            self.ID_i = np.concatenate((self.ID_i, ID), axis=0)\n            self.row = np.concatenate((self.row, row), axis=0)\n            if self.is_train:\n                self.y_i = np.concatenate((self.y_i, y), axis=0)\n            \n            #Save last window for a moving updated window historical data\n            self.last_window = self.X_i[-1]\n            np.savez(pardir+f\"last_window_id{asset_id}_{self.window}.npz\", x=self.last_window)\n        \n            del df, temp\n            gc.collect()\n            \n        #Sort Mini Batch\n        sorter = np.argsort(self.row)\n        #Make sure you are not sorting empty array\n        if len(sorter):\n            self.X_i = self.X_i[sorter]\n            self.ID_i = self.ID_i[sorter]\n        \n            #Append mini batch to full\n            self.X = np.concatenate((self.X.reshape(needed_shape), self.X_i), axis=0)\n            self.ID = np.concatenate((self.ID, self.ID_i), axis=0)\n\n            if self.is_train:\n                self.y_i = self.y_i[sorter]\n                self.y = np.concatenate((self.y, self.y_i), axis=0)\n        \n        del self.X_i, self.ID_i, self.y_i\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:40.019817Z","iopub.execute_input":"2021-11-14T06:01:40.020239Z","iopub.status.idle":"2021-11-14T06:01:40.050978Z","shell.execute_reply.started":"2021-11-14T06:01:40.020208Z","shell.execute_reply":"2021-11-14T06:01:40.050168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class NBatchLogger(tf.keras.callbacks.Callback):\n    \"\"\"\n    A Logger that log average performance per `display` steps.\n    \"\"\"\n    def __init__(self, display):\n        self.step = 0\n        self.display = display\n        self.metric_cache = {}\n\n    def on_batch_end(self, batch, logs={}):\n        self.step += 1\n        for k in self.params['metrics']:\n            if k in logs:\n                self.metric_cache[k] = self.metric_cache.get(k, 0) + logs[k]\n        if self.step % self.display == 0:\n            metrics_log = ''\n            for (k, v) in self.metric_cache.items():\n                val = v / self.display\n                if abs(val) > 1e-3:\n                    metrics_log += ' - %s: %.4f' % (k, val)\n                else:\n                    metrics_log += ' - %s: %.4e' % (k, val)\n            print('step: {}/{} ... {}'.format(self.step,\n                                          self.params['steps'],\n                                          metrics_log))\n            self.metric_cache.clear()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:40.052103Z","iopub.execute_input":"2021-11-14T06:01:40.052412Z","iopub.status.idle":"2021-11-14T06:01:40.069953Z","shell.execute_reply.started":"2021-11-14T06:01:40.052376Z","shell.execute_reply":"2021-11-14T06:01:40.069216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_data_loader = TimeSeriesLoader(\"../input/gresearchcustomfetrain/train_with_FE.csv\")\nclass NPZTrainIterator():\n    def __init__(self, batch_size):\n        self.batch = batch_size\n        self.data = np.load('../input/window-creation/train_data_ready.npz')\n        \n        self.X = self.data['X']\n        self.ID = self.data['ID'].reshape((-1, 1))\n        self.y = self.data['y']\n        self.len = self.y.shape[0]\n        \n        del self.data\n        gc.collect()\n        \n    def __getitem__(self, idx):\n        if type(idx) != int:\n            idx = idx.start #get start of slice\n            \n        if self.len - idx >= self.batch:\n            return ((self.X[idx:idx+self.batch],\n                     self.ID[idx:idx+self.batch]),\n                    self.y[idx:idx+self.batch])\n        else:\n            X = np.zeros((self.batch, WINDOW, 10))\n            ID = np.zeros((self.batch,))\n            y = np.zeros((self.batch))\n            \n            n = self.len - idx\n            X[:n] = self.X[idx:]\n            ID[:n] = self.ID[idx:]\n            y[:n] = self.y[idx:]\n            return ((X, ID), y)\n    \n    def __iter__(self):\n        for i in range(0, self.len, self.batch):\n            (x, i), y = self.__getitem__(i)\n            yield {'fea_in':x, 'emb_in':i}, y\n        return","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:40.071297Z","iopub.execute_input":"2021-11-14T06:01:40.071560Z","iopub.status.idle":"2021-11-14T06:01:40.085125Z","shell.execute_reply.started":"2021-11-14T06:01:40.071527Z","shell.execute_reply":"2021-11-14T06:01:40.084226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"npz = np.load('../input/window-creation/train_data_ready.npz')\nX, ID, y = npz['X'], npz['ID'].reshape(-1, 1), npz['y']\n#get residual\nrmd = X.shape[0]%BATCH_SIZE\nprint(f\"Train rows: {X.shape[0]}, Residual rows: {rmd}, Rows to add: {BATCH_SIZE - rmd}\")\n#Pad data with zeros\nX = np.concatenate([X, np.zeros((BATCH_SIZE-rmd, WINDOW, 10))], axis=0)\nID = np.concatenate([ID, np.zeros((BATCH_SIZE-rmd, 1))], axis=0)\ny = np.concatenate([y, np.zeros((BATCH_SIZE-rmd,))], axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:01:40.086430Z","iopub.execute_input":"2021-11-14T06:01:40.086680Z","iopub.status.idle":"2021-11-14T06:02:22.645503Z","shell.execute_reply.started":"2021-11-14T06:01:40.086653Z","shell.execute_reply":"2021-11-14T06:02:22.644663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Input, LSTM, Dense, Embedding, concatenate, Dropout\n\nseed = 703841\nnp.random.seed(seed)\ntf.random.set_seed(seed)\n\ndef make_model(batch_size=None):\n    fea_in = Input(shape=(WINDOW, 10), batch_size=batch_size, dtype=tf.float64, name='fea_in')\n    feat = LSTM(128, return_sequences=True, dropout=0.1, bias_initializer=\"random_normal\", name='LSTM_fea1')(fea_in)\n    feat = LSTM(128, return_sequences=True, dropout=0.1, bias_initializer=\"random_normal\", name='LSTM_fea2')(feat)\n    feat = LSTM(128, return_sequences=False, bias_initializer=\"random_normal\", name='LSTM_fea3')(feat)\n    feat = Dropout(0.20, name='DropoutFea')(feat)\n    \n    emb_in = Input(shape=(1,), batch_size=batch_size, dtype=tf.int8, name='emb_in')\n    embedding = Embedding(input_dim=15, output_dim=128, name='Embedding')(emb_in)\n    embedding = LSTM(128, return_sequences=True, dropout=0.1, bias_initializer=\"random_normal\", name='LSTM_emb1')(embedding)\n    embedding = LSTM(128, return_sequences=True, dropout=0.1, bias_initializer=\"random_normal\", name='LSTM_emb2')(embedding)\n    embedding = LSTM(128, return_sequences=False, bias_initializer=\"random_normal\", name='LSTM_emb3')(embedding)\n    embedding = Dropout(0.20, name='DropoutEmb')(embedding)\n    \n    pred = concatenate([feat, embedding], name='Concatenate')\n    pred = Dense(1, name='Output')(pred)\n    \n    model = tf.keras.Model(inputs=[fea_in, emb_in], outputs=[pred])\n    return model\n\ntraining_model = make_model(batch_size=BATCH_SIZE)\ntraining_model.summary()\ntf.keras.utils.plot_model(training_model)","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:02:22.646812Z","iopub.execute_input":"2021-11-14T06:02:22.647125Z","iopub.status.idle":"2021-11-14T06:02:25.578583Z","shell.execute_reply.started":"2021-11-14T06:02:22.647093Z","shell.execute_reply":"2021-11-14T06:02:25.577607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_metric(optimizer):\n    def lr(y_true, y_pred):\n        return optimizer._decayed_lr(tf.float32)\n    return lr","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:02:25.580395Z","iopub.execute_input":"2021-11-14T06:02:25.581375Z","iopub.status.idle":"2021-11-14T06:02:25.585409Z","shell.execute_reply.started":"2021-11-14T06:02:25.581333Z","shell.execute_reply":"2021-11-14T06:02:25.584829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = make_model(batch_size=BATCH_SIZE)\n    opt = tf.keras.optimizers.Adam(learning_rate=0.001, decay=1e-5)\n    model.compile(\n        #Train LR will start high on first train loop and become low on further iterations\n        optimizer=opt,\n        loss='mean_squared_error',\n        metrics=['mean_absolute_error', get_lr_metric(opt)])","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:04:32.799597Z","iopub.execute_input":"2021-11-14T06:04:32.799938Z","iopub.status.idle":"2021-11-14T06:04:35.307786Z","shell.execute_reply.started":"2021-11-14T06:04:32.799894Z","shell.execute_reply":"2021-11-14T06:04:35.306840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.display import clear_output\n\nfull_history = []\nstep = BATCH_SIZE*200\nrevs = 10\n\nst = int(np.ceil(X.shape[0]/step))\nfor r in range(revs):\n    for c, i in enumerate(range(0, X.shape[0], step)):\n        clear_output(wait=True)\n        print(f\"Review part {r+1}/{revs}, Training on part {i} - {min(X.shape[0], i+step)} / {X.shape[0]} ({c+1}/{st}):\")\n        hist = model.fit(\n            {'fea_in':X[i:i+step], 'emb_in':ID[i:i+step]}, y[i:i+step],\n            batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1)\n        try:\n            for k in hist.history.keys():\n                full_history[r].history[k] += hist.history[k]\n        except:\n            full_history.append(hist)\n            \nmodel.save_weights(\"./model_weights.hdf5\")","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:04:35.309584Z","iopub.execute_input":"2021-11-14T06:04:35.309845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot each history\nfig, ax = plt.subplots(nrows=1, ncols=2, figsize=(25, 6))\nax[0].title.set_text(\"Train Loss:\")\nax[1].title.set_text(\"Train MAE:\")\nloss_leg = []\nmae_leg = []\nfor i, h in enumerate(full_history):\n    n = len(h.history['loss'])\n    ax[0].plot(range(n), h.history['loss'])\n    loss_leg.append(f\"loss_{i}\")\n    \n    ax[1].plot(range(n), h.history['mean_absolute_error'])\n    mae_leg.append(f\"mae_{i}\")\n    \nax[0].set_ylim([0, 0.001])\nax[0].legend(loss_leg, loc='upper right')\n\nax[1].set_ylim([0, 0.02])\nax[1].legend(mae_leg, loc='upper right')\n\nplt.savefig('loss_and_mae_plots.png')\nplt.show()\nplt.close()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:04:31.298007Z","iopub.status.idle":"2021-11-14T06:04:31.298981Z","shell.execute_reply.started":"2021-11-14T06:04:31.298640Z","shell.execute_reply":"2021-11-14T06:04:31.298677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plot LR history\nlr_leg = []\nfor i, h in enumerate(full_history):\n    n = len(h.history['lr'])\n    plt.plot(range(n), h.history['lr'])\n    lr_leg.append(f\"LR_{i}\")\nplt.ylim([0, 0.001])\nplt.legend(lr_leg)\nplt.title(\"Learning rate decay schedule:\")\n\nplt.savefig(\"lr_decay.png\")\nplt.show()\nplt.close()","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:04:31.300534Z","iopub.status.idle":"2021-11-14T06:04:31.301559Z","shell.execute_reply.started":"2021-11-14T06:04:31.301273Z","shell.execute_reply":"2021-11-14T06:04:31.301301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shutil\n\ntry:\n    os.mkdir('./last_windows/')\nexcept:\n    pass\n\ntgt_dir = '../working/last_windows/'\nfor dirname, _, filenames in os.walk('/kaggle/input/window-creation'):\n    for filename in filenames:\n        f = os.path.join(dirname, filename)\n        if 'last_windows' in f:\n            shutil.copy(f, tgt_dir)\n            print('copied:', f)\n        else:\n            continue","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:04:31.302863Z","iopub.status.idle":"2021-11-14T06:04:31.303336Z","shell.execute_reply.started":"2021-11-14T06:04:31.303079Z","shell.execute_reply":"2021-11-14T06:04:31.303111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Since TPU can't be used on this competition... Prediction is in another notebook! (>,0)/","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}