{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🪙 G-Research Crypto - Starter LGBM Pipeline\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/30894/logos/header.png)\n\nBased on:  https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline-lb-0-164\n \n\n### Just a simple pipeline going from zero to a valid submission\n\nWe train one `LGBMRegressor` for each asset over a very simple set of features, we get the predictions correctly using the iterator and submit. \n\n\n## Please upvote if you find this useful!\n\n## References:\n* [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n* [Basic Submission Template](https://www.kaggle.com/sohier/basic-submission-template)\n* [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)\n\n","metadata":{}},{"cell_type":"markdown","source":"# Import and load dfs\n\nReferences: [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\nfrom tensorflow.python.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\nfrom tensorflow.python.keras.layers import Dropout, LSTM, Dense, LeakyReLU, BatchNormalization\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Sequential\n\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:41:11.954156Z","iopub.execute_input":"2021-11-07T01:41:11.954506Z","iopub.status.idle":"2021-11-07T01:41:11.964919Z","shell.execute_reply.started":"2021-11-07T01:41:11.954471Z","shell.execute_reply":"2021-11-07T01:41:11.963682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(TRAIN_CSV)#,nrows=9234567)\n# df_train = df_train.sort_values([\"timestamp\",\"Asset_ID\"]) # needed if adding lag features (problematic for target)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:41:11.967797Z","iopub.execute_input":"2021-11-07T01:41:11.968713Z","iopub.status.idle":"2021-11-07T01:41:47.896844Z","shell.execute_reply.started":"2021-11-07T01:41:11.968652Z","shell.execute_reply":"2021-11-07T01:41:47.895829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:41:47.898681Z","iopub.execute_input":"2021-11-07T01:41:47.899250Z","iopub.status.idle":"2021-11-07T01:41:47.924476Z","shell.execute_reply.started":"2021-11-07T01:41:47.899202Z","shell.execute_reply":"2021-11-07T01:41:47.923566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"## Utility functions to train a model for one asset","metadata":{}},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df,row=False):\n    df_feat = df[[\"Asset_ID\",'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP',\"timestamp\"]].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    \n    ## Ad dsome more feats\n    df_feat[\"high_div_low\"] = df_feat[\"High\"]/df_feat[\"Low\"]\n    df_feat[\"open_sub_close\"] = df_feat[\"Open\"]-df_feat[\"Close\"]\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df_feat[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n\n    \n    ## todo: features of othet crypto assets in same time period (pivot table + avoid target col)\n    ## Lag of target (be careful to avoid leak? Is this valid for test data (in terms of input format?))\n    ## more features\n    ### time window/time series features\n    return df_feat\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    df_proc = df_proc[~ df_proc.isin([np.nan, np.inf, -np.inf]).any(1)]\n    X = df_proc.drop(\"y\", axis=1)\n    X = X.values\n    y = df_proc[\"y\"].values\n    X_resampled = X.reshape(X.shape[0], 1, X.shape[1])\n    model = Sequential()\n    model.add(LSTM(128, input_shape=(X_resampled.shape[1], X_resampled.shape[2])))\n    model.add(LeakyReLU(alpha=0.03))\n    model.add(BatchNormalization())\n    model.add(Dense(16))\n    model.add(LeakyReLU(alpha=0.03))\n    model.add(BatchNormalization())\n    model.add(Dense(1))\n    optimizer = Adam(learning_rate=0.001)\n    model.compile(loss=\"mean_squared_error\", optimizer=optimizer)\n    model.fit(X_resampled, y, epochs=1, batch_size=512, verbose=1, shuffle=True)\n\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:41:47.927135Z","iopub.execute_input":"2021-11-07T01:41:47.927452Z","iopub.status.idle":"2021-11-07T01:41:47.945578Z","shell.execute_reply.started":"2021-11-07T01:41:47.927408Z","shell.execute_reply":"2021-11-07T01:41:47.944464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loop over all assets","metadata":{}},{"cell_type":"code","source":"%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y, model = get_Xy_and_model_for_asset(df_train, asset_id)    \n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:41:47.947627Z","iopub.execute_input":"2021-11-07T01:41:47.948517Z","iopub.status.idle":"2021-11-07T01:54:28.577878Z","shell.execute_reply.started":"2021-11-07T01:41:47.948420Z","shell.execute_reply":"2021-11-07T01:54:28.576849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Check the model interface\nx = get_features(df_train.iloc[0],row=True).values\nX_resampled = x.reshape(1, 1, x.shape[0])\ny_pred = models[0].predict([X_resampled])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:54:28.579976Z","iopub.execute_input":"2021-11-07T01:54:28.580737Z","iopub.status.idle":"2021-11-07T01:54:28.936988Z","shell.execute_reply.started":"2021-11-07T01:54:28.580691Z","shell.execute_reply":"2021-11-07T01:54:28.936046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict & submit\n\nReferences: [Detailed API Introduction](https://www.kaggle.com/sohier/detailed-api-introduction)\n\nSomething that helped me understand this iterator was adding a pdb checkpoint inside of the for loop:\n\n```python\nimport pdb; pdb.set_trace()\n```\n\nSee [Python Debugging With Pdb](https://realpython.com/python-debugging-pdb/) if you want to use it and you don't know how to.\n","metadata":{"execution":{"iopub.status.busy":"2021-11-02T20:57:49.349459Z","iopub.status.idle":"2021-11-02T20:57:49.349757Z","shell.execute_reply.started":"2021-11-02T20:57:49.349596Z","shell.execute_reply":"2021-11-02T20:57:49.349613Z"}}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():\n        model = models[row['Asset_ID']]\n        x_test = get_features(row,row=True).values\n        X_resampled = x_test.reshape(1, 1, x_test.shape[0])\n        y_pred = model.predict([X_resampled])[0]\n        \n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2021-11-07T01:54:28.938674Z","iopub.execute_input":"2021-11-07T01:54:28.938987Z","iopub.status.idle":"2021-11-07T01:54:29.005332Z","shell.execute_reply.started":"2021-11-07T01:54:28.938941Z","shell.execute_reply":"2021-11-07T01:54:29.004069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}