{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.losses import MeanSquaredError\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-04T08:38:25.718537Z","iopub.execute_input":"2021-11-04T08:38:25.718787Z","iopub.status.idle":"2021-11-04T08:38:30.701823Z","shell.execute_reply.started":"2021-11-04T08:38:25.718706Z","shell.execute_reply":"2021-11-04T08:38:30.701070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv('../input/g-research-crypto-forecasting/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:38:30.705363Z","iopub.execute_input":"2021-11-04T08:38:30.705568Z","iopub.status.idle":"2021-11-04T08:39:22.374550Z","shell.execute_reply.started":"2021-11-04T08:38:30.705543Z","shell.execute_reply":"2021-11-04T08:39:22.373617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.drop(['timestamp'],axis=1)\ndf.replace([np.inf, -np.inf], np.nan, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:22.375753Z","iopub.execute_input":"2021-11-04T08:39:22.376028Z","iopub.status.idle":"2021-11-04T08:39:24.102635Z","shell.execute_reply.started":"2021-11-04T08:39:22.375993Z","shell.execute_reply":"2021-11-04T08:39:24.101863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isna().value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:24.103943Z","iopub.execute_input":"2021-11-04T08:39:24.104227Z","iopub.status.idle":"2021-11-04T08:39:28.616963Z","shell.execute_reply.started":"2021-11-04T08:39:24.104191Z","shell.execute_reply":"2021-11-04T08:39:28.616245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"na_idx=np.unique(list(df.Target[df.Target.isna()==True].index)+list(df.VWAP[df.VWAP.isna()==True].index))","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:28.621712Z","iopub.execute_input":"2021-11-04T08:39:28.623625Z","iopub.status.idle":"2021-11-04T08:39:29.090557Z","shell.execute_reply.started":"2021-11-04T08:39:28.623586Z","shell.execute_reply":"2021-11-04T08:39:29.089770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.drop(na_idx.tolist())","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:29.092100Z","iopub.execute_input":"2021-11-04T08:39:29.092345Z","iopub.status.idle":"2021-11-04T08:39:30.705592Z","shell.execute_reply.started":"2021-11-04T08:39:29.092311Z","shell.execute_reply":"2021-11-04T08:39:30.704830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.Target.isna().value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:30.707019Z","iopub.execute_input":"2021-11-04T08:39:30.707268Z","iopub.status.idle":"2021-11-04T08:39:30.820872Z","shell.execute_reply.started":"2021-11-04T08:39:30.707235Z","shell.execute_reply":"2021-11-04T08:39:30.819964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* There are 14 different pattern time series","metadata":{}},{"cell_type":"code","source":"df.Asset_ID.unique().shape[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:30.822342Z","iopub.execute_input":"2021-11-04T08:39:30.822613Z","iopub.status.idle":"2021-11-04T08:39:30.946961Z","shell.execute_reply.started":"2021-11-04T08:39:30.822577Z","shell.execute_reply":"2021-11-04T08:39:30.946009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(14):\n    plt.plot(range(50),df[df.Asset_ID==i][:50].Target,label=f'Asset:{i}')\n    \nplt.legend()\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:30.948519Z","iopub.execute_input":"2021-11-04T08:39:30.948810Z","iopub.status.idle":"2021-11-04T08:39:34.297325Z","shell.execute_reply.started":"2021-11-04T08:39:30.948773Z","shell.execute_reply":"2021-11-04T08:39:34.296547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs={}\n\nfor i in range(14):\n    dfs[i]=(df[df.Asset_ID==i].drop(['Target','Asset_ID'],axis=1),df[df.Asset_ID==i].Target)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:34.298560Z","iopub.execute_input":"2021-11-04T08:39:34.298856Z","iopub.status.idle":"2021-11-04T08:39:40.735707Z","shell.execute_reply.started":"2021-11-04T08:39:34.298814Z","shell.execute_reply":"2021-11-04T08:39:40.734944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lengths=[]\n\nfor k,v in dfs.items():\n    print(f'length of asset {k} : ',v[0].shape[0])\n    \n    lengths.append(v[0].shape[0])","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:40.736958Z","iopub.execute_input":"2021-11-04T08:39:40.737209Z","iopub.status.idle":"2021-11-04T08:39:40.746435Z","shell.execute_reply.started":"2021-11-04T08:39:40.737177Z","shell.execute_reply":"2021-11-04T08:39:40.745701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* End to End idea \n\n   * Since RNN pass zero initial hidden input ,doing preorder padding to the same length\n   \n   * Use asset_id to representation which asset \n   \n   * Each asset series size : (1,seq slice length , dim)\n   \n       * So total size will be : (14,   seq slice length , dim)\n       \n   * When Using lstm or gru , pass last hidden output to next initial hidden input to retain time dependency\n   ","metadata":{}},{"cell_type":"code","source":"max_len=np.max(lengths)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:40.747434Z","iopub.execute_input":"2021-11-04T08:39:40.747708Z","iopub.status.idle":"2021-11-04T08:39:40.754080Z","shell.execute_reply.started":"2021-11-04T08:39:40.747671Z","shell.execute_reply":"2021-11-04T08:39:40.753229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this operation is very memory intensive \n\nx=[]\ny=[]\n\nfor k,v in dfs.items():\n    print(k)\n    x.append(np.array(v[0]))\n    y.append(np.array(v[1]))","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:40.755572Z","iopub.execute_input":"2021-11-04T08:39:40.756080Z","iopub.status.idle":"2021-11-04T08:39:42.052169Z","shell.execute_reply.started":"2021-11-04T08:39:40.756042Z","shell.execute_reply":"2021-11-04T08:39:42.051336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this operation is very memory intensive \nx=pad_sequences(x,dtype='float32')","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:42.055615Z","iopub.execute_input":"2021-11-04T08:39:42.056292Z","iopub.status.idle":"2021-11-04T08:39:43.245264Z","shell.execute_reply.started":"2021-11-04T08:39:42.056250Z","shell.execute_reply":"2021-11-04T08:39:43.244507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x.shape #14 subbatch , 1955978 sequence length , each have 8 dimension","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.246428Z","iopub.execute_input":"2021-11-04T08:39:43.246674Z","iopub.status.idle":"2021-11-04T08:39:43.251764Z","shell.execute_reply.started":"2021-11-04T08:39:43.246644Z","shell.execute_reply":"2021-11-04T08:39:43.251104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=pad_sequences(y,dtype='float32')","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.253268Z","iopub.execute_input":"2021-11-04T08:39:43.253680Z","iopub.status.idle":"2021-11-04T08:39:43.332035Z","shell.execute_reply.started":"2021-11-04T08:39:43.253644Z","shell.execute_reply":"2021-11-04T08:39:43.331293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape #14 subbatch, 1955978 sequence length","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.333435Z","iopub.execute_input":"2021-11-04T08:39:43.333765Z","iopub.status.idle":"2021-11-04T08:39:43.341199Z","shell.execute_reply.started":"2021-11-04T08:39:43.333711Z","shell.execute_reply":"2021-11-04T08:39:43.340463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.342475Z","iopub.execute_input":"2021-11-04T08:39:43.343119Z","iopub.status.idle":"2021-11-04T08:39:43.349805Z","shell.execute_reply.started":"2021-11-04T08:39:43.343069Z","shell.execute_reply":"2021-11-04T08:39:43.348944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x=np.transpose(x,[1,0,2])\ny=np.transpose(y,[1,0])","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.351427Z","iopub.execute_input":"2021-11-04T08:39:43.351692Z","iopub.status.idle":"2021-11-04T08:39:43.357047Z","shell.execute_reply.started":"2021-11-04T08:39:43.351660Z","shell.execute_reply":"2021-11-04T08:39:43.356173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x.shape,y.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.358609Z","iopub.execute_input":"2021-11-04T08:39:43.358904Z","iopub.status.idle":"2021-11-04T08:39:43.367510Z","shell.execute_reply.started":"2021-11-04T08:39:43.358869Z","shell.execute_reply":"2021-11-04T08:39:43.366679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{}},{"cell_type":"code","source":"mmin=x.min(axis=0)\nmmax=x.max(axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:43.369060Z","iopub.execute_input":"2021-11-04T08:39:43.370049Z","iopub.status.idle":"2021-11-04T08:39:44.930592Z","shell.execute_reply.started":"2021-11-04T08:39:43.370011Z","shell.execute_reply":"2021-11-04T08:39:44.929774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def norm(x):\n    return (x-mmin)/(mmax-mmin+1)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:44.931861Z","iopub.execute_input":"2021-11-04T08:39:44.932134Z","iopub.status.idle":"2021-11-04T08:39:44.937474Z","shell.execute_reply.started":"2021-11-04T08:39:44.932101Z","shell.execute_reply":"2021-11-04T08:39:44.936784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mmin.shape,mmax.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:44.938943Z","iopub.execute_input":"2021-11-04T08:39:44.939440Z","iopub.status.idle":"2021-11-04T08:39:44.948560Z","shell.execute_reply.started":"2021-11-04T08:39:44.939389Z","shell.execute_reply":"2021-11-04T08:39:44.947557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_ds(X,y,time_slice,batch_size):\n    \n    ds_X=tf.keras.preprocessing.timeseries_dataset_from_array(\n      data=X,\n      targets=tf.ones((X.shape[0])),\n      sequence_length=time_slice,\n      sequence_stride=time_slice,\n      shuffle=False,\n      batch_size=batch_size\n    ).map(lambda x,y : x)\n\n    ds_Y=tf.keras.preprocessing.timeseries_dataset_from_array(\n      data=y,\n      targets=tf.ones((y.shape[0])),\n      sequence_length=time_slice,\n      sequence_stride=time_slice,\n      shuffle=False,\n      batch_size=batch_size\n    ).map(lambda x,y : x)\n    ds=tf.data.Dataset.zip((ds_X,ds_Y))\n    ds=ds.map(lambda x,y:(norm(x),y)) #broadcast\n    ds=ds.map(lambda x,y:(tf.transpose(x,perm=[0,2,1,3]),tf.transpose(y,perm=[0,2,1])))\n    ds=ds.prefetch(tf.data.experimental.AUTOTUNE)\n    return ds","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:44.950090Z","iopub.execute_input":"2021-11-04T08:39:44.950422Z","iopub.status.idle":"2021-11-04T08:39:44.961153Z","shell.execute_reply.started":"2021-11-04T08:39:44.950385Z","shell.execute_reply":"2021-11-04T08:39:44.960370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Hyperparameter","metadata":{}},{"cell_type":"code","source":"time_slice=1024\nbatch_size=128\n\nepochs=3\nunits=512\nlr=0.0001","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:44.962358Z","iopub.execute_input":"2021-11-04T08:39:44.962685Z","iopub.status.idle":"2021-11-04T08:39:44.972385Z","shell.execute_reply.started":"2021-11-04T08:39:44.962644Z","shell.execute_reply":"2021-11-04T08:39:44.971760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds=build_ds(x,y,time_slice,batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:44.974641Z","iopub.execute_input":"2021-11-04T08:39:44.975263Z","iopub.status.idle":"2021-11-04T08:39:49.676626Z","shell.execute_reply.started":"2021-11-04T08:39:44.975225Z","shell.execute_reply":"2021-11-04T08:39:49.675906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for seq,label in ds:\n    print(seq.shape,label.shape)  #x:(batch,seq length, sub batch ,dim)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:49.677897Z","iopub.execute_input":"2021-11-04T08:39:49.678298Z","iopub.status.idle":"2021-11-04T08:39:51.289336Z","shell.execute_reply.started":"2021-11-04T08:39:49.678261Z","shell.execute_reply":"2021-11-04T08:39:51.288580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check operation bug\n\ntf.reduce_sum(tf.cast(tf.math.is_nan(seq),'float32'))","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:39:51.290936Z","iopub.execute_input":"2021-11-04T08:39:51.291270Z","iopub.status.idle":"2021-11-04T08:39:51.331576Z","shell.execute_reply.started":"2021-11-04T08:39:51.291231Z","shell.execute_reply":"2021-11-04T08:39:51.330913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Model\n\n   * Stateful : Boolean (default False). If True, the last state for each sample at index i in a batch will be used as initial state for the sample of index i in the following batch\n   \n   * If lstm is stateful , we need to specify batch input shape in the input layer","metadata":{}},{"cell_type":"code","source":"class Model(tf.keras.Model):\n    def __init__(self,units,batch_size=14,dim=7):\n        super().__init__()\n        self.batch_size=batch_size\n        self.units=units\n        self.mask=layers.Masking(batch_input_shape=(batch_size, time_slice,dim))\n        self.lstm=layers.LSTM(units=units,\n                            return_sequences=True,\n                            return_state=True,\n                            dropout=0.3)\n        self.d1=layers.Dense(units=int(units/2),activation='relu')\n        self.d2=layers.Dense(units=1,activation=None)\n        \n    def call(self,x,hidden=None,training=False):\n        x=self.mask(x)\n        if hidden==None:\n            x, h, c=self.lstm(x,initial_state =self.initialize_hidden_state(),training=training)\n        else:\n            x, h, c=self.lstm(x,initial_state=hidden,training=training)\n        x=self.d1(x)\n        x=self.d2(x)\n        return x,h, c\n    \n    \n    def  initialize_hidden_state(self):\n        return [tf.zeros((self.batch_size, self.units)), tf.zeros((self.batch_size, self.units))]\n        ","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:40:33.936086Z","iopub.execute_input":"2021-11-04T08:40:33.936352Z","iopub.status.idle":"2021-11-04T08:40:33.948272Z","shell.execute_reply.started":"2021-11-04T08:40:33.936324Z","shell.execute_reply":"2021-11-04T08:40:33.946190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@tf.function\ndef flow(x,y,model,loss_func,opt,init=None,):\n    with tf.GradientTape() as tape:\n        y_pred,h,c=model(x,hidden=init,training=True)\n        loss=loss_func(y,y_pred)\n    grad=tape.gradient(loss,model.trainable_weights)\n    opt.apply_gradients(zip(grad,model.trainable_weights))\n    \n    return loss,h,c","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:40:43.087657Z","iopub.execute_input":"2021-11-04T08:40:43.088372Z","iopub.status.idle":"2021-11-04T08:40:43.094449Z","shell.execute_reply.started":"2021-11-04T08:40:43.088333Z","shell.execute_reply":"2021-11-04T08:40:43.093360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train():\n    model=Model(units)\n    opt=tf.keras.optimizers.Adam(learning_rate=lr)\n    mse=MeanSquaredError()\n    ckpt = tf.train.Checkpoint(opt=opt, model=model)\n    manager = tf.train.CheckpointManager(ckpt, './ckpt', max_to_keep=1)\n    \n    print('start training')\n    for epoch in range(epochs):\n        init=None\n        for i,(x,y) in enumerate(ds): #batch\n            for j,(seq,label) in enumerate(zip(x,y)):\n                label=tf.expand_dims(label,axis=-1)\n                loss,h,c=flow(seq,label,model,mse,opt,init=init)\n                init=[h,c]\n                if j%50==0:\n                    print(f'epoch :{epoch} , batch:{i} sub: {j} , loss={loss}')\n        manager.save()\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:40:48.828139Z","iopub.execute_input":"2021-11-04T08:40:48.828923Z","iopub.status.idle":"2021-11-04T08:40:48.837989Z","shell.execute_reply.started":"2021-11-04T08:40:48.828873Z","shell.execute_reply":"2021-11-04T08:40:48.836737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=train()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T08:40:51.224312Z","iopub.execute_input":"2021-11-04T08:40:51.224903Z","iopub.status.idle":"2021-11-04T09:32:12.279666Z","shell.execute_reply.started":"2021-11-04T08:40:51.224864Z","shell.execute_reply":"2021-11-04T09:32:12.278890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-04T09:39:06.387104Z","iopub.execute_input":"2021-11-04T09:39:06.387689Z","iopub.status.idle":"2021-11-04T09:39:06.397017Z","shell.execute_reply.started":"2021-11-04T09:39:06.387654Z","shell.execute_reply":"2021-11-04T09:39:06.396294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"markdown","source":"*  Predict each data once , passing hidden state to next batch","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\n\nh,c=0,0\nenv = gresearch_crypto.make_env()  \niter_test = env.iter_test()    \nmodel.reset_states() #h0-->h1-->h2--->h3\nfor i,(test_df, sample_prediction_df) in enumerate(iter_test):\n    \n    test_df=test_df.drop(['timestamp','row_id','Asset_ID'],axis=1)\n    k=tf.reshape(test_df,(test_df.shape[0],1,test_df.shape[1]))\n    mmax_=tf.cast(tf.reshape(mmax,(14,1,7)),'float64')\n    mmin_=tf.cast(tf.reshape(mmin,(14,1,7)),'float64')\n    k=(k-mmin_)/(mmax_-mmin_+1)\n    \n    y_pred,h,c=model(k,hidden=None if i==0 else [h,c],training=False)\n    sample_prediction_df['Target']=tf.cast(y_pred,'float64')[:,0,0]\n    env.predict(sample_prediction_df)   ","metadata":{"execution":{"iopub.status.busy":"2021-11-04T09:39:09.556222Z","iopub.execute_input":"2021-11-04T09:39:09.556770Z","iopub.status.idle":"2021-11-04T09:39:09.664259Z","shell.execute_reply.started":"2021-11-04T09:39:09.556727Z","shell.execute_reply":"2021-11-04T09:39:09.663516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}