{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective :-\n\nHere we have used an autoencoder like artitecture to create more features take for example there is a input data X and a target y .  So the neural network will try to map a linear function it will take input X , and also the target will be X or the input itself something like f(x)=x where f(x) is the model a particular hidden layer of the network will generate a 64 dimensional output which is our 64 new features You can find the model or the output file here :- https://www.kaggle.com/swaralipibose/autoencoderfeat it is a keras ann and is stored as'model {asset_id}.h5' format the model has also been trained with some extra features in ```get_features``` function so basically model.predict(get_features(inputs)).shape=(n,64). If you want the new features generated for ```train.csv``` [ ```test.csv``` is not provided and you may need to compute that using the models] they are stored .npy format '{asset_id}.npy' is the name you can find it here https://www.kaggle.com/swaralipibose/autoencoderfeatures the notebook covers the training of the autoencoder :-)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-11-12T15:21:17.80166Z","iopub.execute_input":"2021-11-12T15:21:17.802221Z","iopub.status.idle":"2021-11-12T15:21:20.560615Z","shell.execute_reply.started":"2021-11-12T15:21:17.802127Z","shell.execute_reply":"2021-11-12T15:21:20.559852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-12T14:53:15.431623Z","iopub.execute_input":"2021-11-12T14:53:15.431877Z","iopub.status.idle":"2021-11-12T14:53:16.45393Z","shell.execute_reply.started":"2021-11-12T14:53:15.431848Z","shell.execute_reply":"2021-11-12T14:53:16.453171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-12T14:28:31.586673Z","iopub.execute_input":"2021-11-12T14:28:31.586986Z","iopub.status.idle":"2021-11-12T14:28:31.611595Z","shell.execute_reply.started":"2021-11-12T14:28:31.586944Z","shell.execute_reply":"2021-11-12T14:28:31.610877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## Utility functions to train a model for one asset","metadata":{}},{"cell_type":"markdown","source":"## Loop over all assets","metadata":{}},{"cell_type":"code","source":"from scipy.stats import pearsonr\nfrom sklearn.model_selection import train_test_split\nfrom keras.layers import *\nfrom keras import *\nfrom tensorflow.keras.optimizers import *\n# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"high_div_low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    #df_feat[\"open_sub_close\"] = df_feat[\"Open\"] - df_feat[\"Close\"]\n    df_feat['trade']=df_feat['Close']-df_feat['Open']\n    df_feat['gtrade']=df_feat['trade']/df_feat['Count']\n    df_feat['shadow1']=df_feat['trade']/df_feat['Volume']\n    #df_feat['shadow2']=df_feat['upper_Shadow']/df['Low']\n    df_feat['shadow3']=df_feat['upper_Shadow']/df_feat['Volume']\n    #df_feat['shadow4']=df_feat['lower_Shadow']/df['High']\n    df_feat['shadow5']=df_feat['lower_Shadow']/df_feat['Volume']\n    \n    df_feat['diff1']=df_feat['Volume']-df_feat['Count']\n    \n    df_feat['mean1']=(df_feat['shadow5']+df_feat['shadow3'])/2\n    \n    df_feat['mean2']=(df_feat['shadow1']+df_feat['Volume'])/2\n    \n    df_feat['mean3']=(df_feat['trade']+df_feat['gtrade'])/2\n    \n    df_feat['mean4']=(df_feat['diff1']+df_feat['upper_Shadow'])/2\n    \n    df_feat['mean5']=(df_feat['diff1']+df_feat['lower_Shadow'])/2 \n    \n    return df_feat\ndef log(model,X_train, X_valid, y_train, y_valid,train_split=1.0):\n    if train_split > 0:\n        X_train=X_train[:int(train_split*X_train.shape[0])]\n        y_train=y_train[:int(train_split*y_train.shape[0])]\n    \n        pred=model.predict(X_train)\n        print('Training :- ')\n        print(f'MSE : {np.mean((y_train-pred)**2)}')\n        print(f'CV : {pearsonr(pred,y_train)[0]}')\n    pred=model.predict(X_valid)\n    print('Validation :- ')\n    print(f'MSE : {np.mean((y_valid-pred)**2)}')\n    print(f'CV : {pearsonr(pred,y_valid)[0]}')\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n   \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    X=np.array(X)\n    \n    input=Input(shape=X.shape[1:])\n\n    x=Dense(64,activation=None,name='end')(input)\n    \n    x=Dense(X.shape[1])(x)\n    model=Model(input,x)\n    model.compile(loss='mae',optimizer=Adam(0.0001),metrics=['mae'])\n    model.fit(X,X,epochs=12,batch_size=1,validation_split=0.2)\n    Model(model.input,model.get_layer('end').output).save(f'model {asset_id}.h5')\n    ","metadata":{"execution":{"iopub.status.busy":"2021-11-12T14:33:00.676315Z","iopub.execute_input":"2021-11-12T14:33:00.67681Z","iopub.status.idle":"2021-11-12T14:33:00.695185Z","shell.execute_reply.started":"2021-11-12T14:33:00.676769Z","shell.execute_reply":"2021-11-12T14:33:00.694148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    get_Xy_and_model_for_asset(df_train, asset_id)    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-11-12T14:33:01.18929Z","iopub.execute_input":"2021-11-12T14:33:01.190187Z","iopub.status.idle":"2021-11-12T14:36:00.193165Z","shell.execute_reply.started":"2021-11-12T14:33:01.190134Z","shell.execute_reply":"2021-11-12T14:36:00.191843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}