{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#all imports\nfrom sklearn.preprocessing import StandardScaler\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import Sequential, layers, callbacks\nfrom tensorflow.keras.layers import Dense, LSTM, Dropout\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For training the models, my initial plan was to run for 10 epochs.However the dataset is huge. Opting for less number of epochs and using learning rate scheduling is a better compromise and most importantly it saves time.\nTo get to know more about learning rate scheduling refer:\nhttps://machinelearningmastery.com/understand-the-dynamics-of-learning-rate-on-deep-learning-neural-networks/","metadata":{}},{"cell_type":"code","source":"#lr scheduling \nimport math\ndef step_decay(epoch):\n    initial_lrate = 0.1\n    epochs_drop = 10.0\n    lrate = initial_lrate * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n    return lrate","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This function takes care of removing outliers using quantile capping.","metadata":{}},{"cell_type":"code","source":"def quantile_capping(df):\n    \n    for col in df.columns:\n        percentiles = df[col].quantile([0.01, 0.90]).values\n        df[col][df[col] <= percentiles[0]] = percentiles[0]\n        df[col][df[col] >= percentiles[1]] = percentiles[1]\n    return df    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"log_return function takes an series i.e one paticular column from our input data, obtains logarithmic values and finds the difference based on periods mentioned. Same implemenation as that shown in tutorial notebook. However I applied np.abs(series) to remove negative log undefined values","metadata":{}},{"cell_type":"code","source":"#log difference\ndef log_return(series, periods=1):\n    return np.log(np.abs(series)).diff(periods=periods)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"create_model_0to9 - Models for asset_id 0 to 9\n\ncreate_model_10to13 - Models for asset_id 10 to 13\n\nWhy different models? The last set of models did not converge. I made some changes like activation function and gradient clipping. But the loss value was NAN. If you can suggest some fixes, please write down in the comment section below.","metadata":{}},{"cell_type":"code","source":"#lstm model\ndef create_model_0to9(X):\n    model = Sequential()\n    # First layer of LSTM\n    model.add(LSTM(64, return_sequences = True, \n                 input_shape = [X.shape[1], X.shape[2]]))\n    model.add(Dropout(0.2)) \n    # Second layer of LSTM\n    model.add(LSTM(64))                 \n    model.add(Dropout(0.2))\n    model.add(Dense(units = 1)) \n    #Compile model\n    model.compile(loss='mse', optimizer='adam')\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model_10to13(X):\n    model = Sequential()\n    # First layer of LSTM\n    model.add(LSTM(64, return_sequences = True, activation=None,recurrent_activation=None,\n                 input_shape = [X.shape[1], X.shape[2]]))\n    model.add(Dropout(0.2)) \n    # Second layer of LSTM\n    model.add(LSTM(64))                 \n    model.add(Dropout(0.2))\n    model.add(Dense(units = 1 ,activation=None)) \n    opt = tf.keras.optimizers.SGD(lr=0.01, momentum=0.9, clipvalue=5.0)\n    #Compile model\n    #gradient clipping\n    model.compile(loss='mse', optimizer=opt)\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Feature Extraction- same as that implemented in tutorial notebook","metadata":{}},{"cell_type":"code","source":"upper_shadow = lambda asset: asset.High - np.maximum(asset.Close,asset.Open)\nlower_shadow = lambda asset: np.minimum(asset.Close,asset.Open)- asset.Low","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we go!\nThe training commences for each and every asset_id\nSummary of the process-\n1. Train data chosen till timestamp 162354200. The test data example starts from there. Avoiding data leakage.\n2. Removing missing values,replacing 0 with 1 to avoid log 0 undefined error\n3. Scaler to Normalise input values\n4. Reshape- LSTM models take 3d datasets. To know more visit:-https://machinelearningmastery.com/reshape-input-data-long-short-term-memory-networks-keras/\n5.Save model for future use. Copy the link from \"Copy File Path\" in Output Section beside \"/kaggle/working\"\n","metadata":{}},{"cell_type":"code","source":"#build one model for each asset_id\n\nfor i in range(0,10):#model assetid 0 to 10\n    train_data=df_train[df_train[\"Asset_ID\"]==i]\n    train_data=train_data[train_data[\"timestamp\"]<1623542400]\n    train_data[\"Target\"].fillna(train_data[\"Target\"].interpolate(),inplace=True)\n    X_c=pd.concat([log_return(train_data.VWAP,periods=5), log_return(train_data.VWAP,periods=1).abs(), \n               upper_shadow(train_data), lower_shadow(train_data)], axis=1)\n    y_c = train_data.Target\n    X_c.fillna(1,inplace=True)\n    X_c.replace(0,1)# to avoid log 0 error\n    X_c.columns=['VWAP-5','VWAP-1','UpperShadow','LowerShadow']\n    X_c.reset_index(drop=True,inplace=True)\n    X_c=quantile_capping(X_c)\n    scaler = StandardScaler()\n    X_c_scaled = scaler.fit_transform(X_c)\n    X_c_scaled=X_c_scaled.reshape(X_c_scaled.shape[0],X_c_scaled.shape[1],1)\n    model=create_model_0to9(X_c_scaled)\n    lrate = LearningRateScheduler(step_decay)\n    callbacks_list = [lrate]\n    model.fit(X_c_scaled,   y_c, epochs = 5, validation_split = 0.2,\n                    batch_size = 32, shuffle = False, callbacks =[callbacks_list])\n    model.save('./model_asset_id{}.h5'.format(i))\n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(10,14):#model assetid 10 to 13\n    train_data=df_train[df_train[\"Asset_ID\"]==i]\n    train_data=train_data[train_data[\"timestamp\"]<1623542400]\n    train_data[\"Target\"].fillna(train_data[\"Target\"].interpolate(),inplace=True)\n    X_c=pd.concat([log_return(train_data.VWAP,periods=5), log_return(train_data.VWAP,periods=1).abs(), \n               upper_shadow(train_data), lower_shadow(train_data)], axis=1)\n    y_c = train_data.Target\n    y_c=y_c.interpolate()\n    X_c.fillna(1,inplace=True)\n    X_c.replace(0,1)# to avoid log 0 error\n    X_c.columns=['VWAP-5','VWAP-1','UpperShadow','LowerShadow']\n    X_c.reset_index(drop=True,inplace=True)\n    X_c=quantile_capping(X_c)\n    scaler = StandardScaler()\n    X_c_scaled = scaler.fit_transform(X_c)\n    X_c_scaled=X_c_scaled.reshape(X_c_scaled.shape[0],X_c_scaled.shape[1],1)\n    model=create_model_10to13(X_c_scaled)\n    lrate = LearningRateScheduler(step_decay)\n    callbacks_list = [lrate]\n    model.fit(X_c_scaled,   y_c, epochs = 5, validation_split = 0.2,\n                    batch_size = 32, shuffle = False, callbacks =[callbacks_list])\n    model.save('./model_asset_id{}.h5'.format(i))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preprocessing of the test dataset to be executed in the same way as example_test_df\ndf_test=pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')\n#df_test['Target']=0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission=pd.read_csv('../input/g-research-crypto-forecasting/example_sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Creating a copy of df_submission because the original file is in read only format. ","metadata":{}},{"cell_type":"code","source":"submission_df_op = pd.DataFrame(columns=['group_num','row_id','Target'])\nsubmission_df_op['group_num']=df_submission['group_num']\nsubmission_df_op['row_id']=df_submission['row_id']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Testing starts here-","metadata":{}},{"cell_type":"code","source":"for i in range(0,14):\n    test_data=df_test[df_test[\"Asset_ID\"]==i]\n    X_c=pd.concat([log_return(test_data.VWAP,periods=5), log_return(test_data.VWAP,periods=1).abs(), \n              upper_shadow(test_data), lower_shadow(test_data)], axis=1)\n   \n    X_c.fillna(1,inplace=True)\n    X_c.replace(0,1)# to avoid log 0 error\n    X_c.columns=['VWAP-5','VWAP-1','UpperShadow','LowerShadow']\n    X_c.reset_index(drop=True,inplace=True)\n    scaler = StandardScaler()\n    X_c_scaled = scaler.fit_transform(X_c)\n    X_c_scaled=X_c_scaled.reshape(X_c_scaled.shape[0],X_c_scaled.shape[1],1)\n    model=keras.models.load_model('./model_asset_id{}.h5'.format(i))\n    y=model.predict(X_c_scaled)\n    rid=test_data[\"row_id\"].to_list()\n    k=0\n    index=0\n    for j in range(0,submission_writeop.shape[0]):\n        try :\n            index=rid.index(df_submission['row_id'][j])\n            submission_df_op.at[j,'Target']=y[k][0]\n            \n        except ValueError as e:\n            continue\n        k=k+1    \n    \n    \n    \n    \n    \n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Merge the dataframes and obtain rmse score","metadata":{}},{"cell_type":"code","source":"#sample code\n#df_predandactual=pd.merge(df_pred_dftest,df_train,on='timestamp',how='inner')\n#from sklearn.metrics import mean_squared_error\n#mean_squared_error(df_predandactual['Target_x'], df_predandactual['Target_y'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The comparision was done with models 0 to 9 and 13, the results obtained and the actual values\nReceived RMSE score-3.4427147667227486e-05","metadata":{}},{"cell_type":"code","source":"\nsubmission_df_op.to_csv('./output.csv',index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}