{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-27T15:20:02.720804Z","iopub.execute_input":"2021-11-27T15:20:02.721721Z","iopub.status.idle":"2021-11-27T15:20:06.250177Z","shell.execute_reply.started":"2021-11-27T15:20:02.721626Z","shell.execute_reply":"2021-11-27T15:20:06.248953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:20:06.254641Z","iopub.execute_input":"2021-11-27T15:20:06.254916Z","iopub.status.idle":"2021-11-27T15:21:03.378975Z","shell.execute_reply.started":"2021-11-27T15:20:06.254879Z","shell.execute_reply":"2021-11-27T15:21:03.378103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:21:03.380558Z","iopub.execute_input":"2021-11-27T15:21:03.380952Z","iopub.status.idle":"2021-11-27T15:21:03.409999Z","shell.execute_reply.started":"2021-11-27T15:21:03.380899Z","shell.execute_reply":"2021-11-27T15:21:03.409121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Features","metadata":{}},{"cell_type":"markdown","source":"Eda on these features :- https://www.kaggle.com/swaralipibose/new-features-eda-using-elon-musk-and-crypto-trends","metadata":{}},{"cell_type":"code","source":"def upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef add_features(dataframe):\n    try:\n        dataframe=dataframe[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','timestamp','Target']]\n    except:\n        dataframe=dataframe[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP','timestamp']]\n        \n        \n    \n    dataframe['dat']=pd.to_datetime(dataframe['timestamp'],unit='s')\n    dataframe['date']=dataframe['dat'].dt.date\n    dataframe[\"Day of Week\"] = dataframe['dat'].dt.dayofweek\n    dataframe['weekend']=np.where(dataframe['Day of Week']>4,1,0)\n    date_of_elon_musk_tweet='2018-6-04'\n    dataframe['after']=(dataframe['dat']>date_of_elon_musk_tweet).astype(int)\n    date_of_elon_musk_tweet='2021-03-29'\n    dataframe['afterdoge']=(dataframe['dat']>date_of_elon_musk_tweet).astype(int)\n    dataframe['Month']=dataframe['dat'].dt.month\n    dataframe['upper_Shadow'] = upper_shadow(dataframe)\n    dataframe['lower_Shadow'] = lower_shadow(dataframe)\n    dataframe[\"high_div_low\"] = dataframe[\"High\"] / dataframe[\"Low\"]\n    #dataframe[\"open_sub_close\"] = dataframe[\"Open\"] - dataframe[\"Close\"]\n    dataframe['trade']=dataframe['Close']-dataframe['Open']\n    dataframe['gtrade']=dataframe['trade']/dataframe['Count']\n    dataframe['shadow1']=dataframe['trade']/dataframe['Volume']\n    #dataframe['shadow2']=dataframe['upper_Shadow']/df['Low']\n    dataframe['shadow3']=dataframe['upper_Shadow']/dataframe['Volume']\n    #dataframe['shadow4']=dataframe['lower_Shadow']/dataframe['High']\n    dataframe['shadow5']=dataframe['lower_Shadow']/dataframe['Volume']\n    \n \n    return dataframe.drop(columns=['dat','date','timestamp'])","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:21:03.412667Z","iopub.execute_input":"2021-11-27T15:21:03.413143Z","iopub.status.idle":"2021-11-27T15:21:03.426656Z","shell.execute_reply.started":"2021-11-27T15:21:03.413059Z","shell.execute_reply":"2021-11-27T15:21:03.425511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training and feature importance","metadata":{}},{"cell_type":"code","source":"import lightgbm","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:21:03.429170Z","iopub.execute_input":"2021-11-27T15:21:03.429546Z","iopub.status.idle":"2021-11-27T15:21:03.438169Z","shell.execute_reply.started":"2021-11-27T15:21:03.429501Z","shell.execute_reply":"2021-11-27T15:21:03.436932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import pearsonr\nfrom sklearn.model_selection import train_test_split\n\ndef log(model,X_train, X_valid, y_train, y_valid,train_split=1.0):\n    if train_split > 0:\n        X_train=X_train[:int(train_split*X_train.shape[0])]\n        y_train=y_train[:int(train_split*y_train.shape[0])]\n    \n        pred=model.predict(X_train)\n        print('Training :- ')\n        print(f'MSE : {np.mean((y_train-pred)**2)}')\n        print(f'CV : {pearsonr(pred,y_train)[0]}')\n    pred=model.predict(X_valid)\n    print('Validation :- ')\n    print(f'MSE : {np.mean((y_valid-pred)**2)}')\n    print(f'CV : {pearsonr(pred,y_valid)[0]}')\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df_proc = add_features(df_train[df_train[\"Asset_ID\"] == asset_id])\n   \n    # TODO: Try different features here!\n\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"Target\", axis=1)\n    y = df_proc[\"Target\"]\n    X_train=X[:int(0.7*X.shape[0])]\n    y_train=y[:int(0.7*y.shape[0])]#\n    X_test=X[int(X.shape[0]*0.7):]\n    y_test=y[int(y.shape[0]*0.7):]\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=200,num_leaves=300,learning_rate=0.09)\n    model.fit(X_train, y_train)\n    print('[Finished Training] evaluating')\n    log(model,X_train, X_test, y_train, y_test,0.3)\n    \n    print(X_train.columns)\n    \n    ax=lightgbm.plot_importance(model)\n    plt.savefig(f'{asset_id}.png')\n    \n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:29:25.763102Z","iopub.execute_input":"2021-11-27T15:29:25.763448Z","iopub.status.idle":"2021-11-27T15:29:25.777118Z","shell.execute_reply.started":"2021-11-27T15:29:25.763410Z","shell.execute_reply":"2021-11-27T15:29:25.775571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:29:25.990436Z","iopub.execute_input":"2021-11-27T15:29:25.990674Z","iopub.status.idle":"2021-11-27T15:29:25.994931Z","shell.execute_reply.started":"2021-11-27T15:29:25.990646Z","shell.execute_reply":"2021-11-27T15:29:25.993915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    models[asset_id] = model","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:29:26.232711Z","iopub.execute_input":"2021-11-27T15:29:26.233037Z","iopub.status.idle":"2021-11-27T15:37:40.676064Z","shell.execute_reply.started":"2021-11-27T15:29:26.233004Z","shell.execute_reply":"2021-11-27T15:37:40.674941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details=pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nasset_details.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:37:40.678561Z","iopub.execute_input":"2021-11-27T15:37:40.679645Z","iopub.status.idle":"2021-11-27T15:37:40.701352Z","shell.execute_reply.started":"2021-11-27T15:37:40.679604Z","shell.execute_reply":"2021-11-27T15:37:40.700241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dic=dict(zip(asset_details['Asset_ID'],asset_details['Asset_Name']))","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:39:32.873172Z","iopub.execute_input":"2021-11-27T15:39:32.873560Z","iopub.status.idle":"2021-11-27T15:39:32.882190Z","shell.execute_reply.started":"2021-11-27T15:39:32.873504Z","shell.execute_reply":"2021-11-27T15:39:32.879625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2\nsubplots=plt.subplots(14,1,figsize=(100,100))[1]\nfor asset in range(14):\n    img=cv2.imread(f'{asset}.png')\n    subplots[asset].imshow(img)\n    subplots[asset].text(150,15,dic[asset])\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:41:22.583988Z","iopub.execute_input":"2021-11-27T15:41:22.584367Z","iopub.status.idle":"2021-11-27T15:41:26.322671Z","shell.execute_reply.started":"2021-11-27T15:41:22.584322Z","shell.execute_reply":"2021-11-27T15:41:26.321034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model interface\nx = add_features(df_train.iloc[[1,2]]).drop(columns=['Target']\n                                        )\ny_pred = models[0].predict(x)\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:41:43.168679Z","iopub.execute_input":"2021-11-27T15:41:43.169036Z","iopub.status.idle":"2021-11-27T15:41:43.201652Z","shell.execute_reply.started":"2021-11-27T15:41:43.169006Z","shell.execute_reply":"2021-11-27T15:41:43.200022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"''' \nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_test=add_features(df_test)\n    for j , row in df_test.iterrows():\n        \n        model = models[row['Asset_ID']]\n        x_test = row\n        y_pred = model.predict([x_test])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)\n''' ","metadata":{"execution":{"iopub.status.busy":"2021-11-27T15:41:47.461630Z","iopub.execute_input":"2021-11-27T15:41:47.462392Z","iopub.status.idle":"2021-11-27T15:41:47.531147Z","shell.execute_reply.started":"2021-11-27T15:41:47.462352Z","shell.execute_reply":"2021-11-27T15:41:47.529974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}