{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0\"></a>\n# Competition \"[G-Research Crypto Forecasting](https://www.kaggle.com/c/g-research-crypto-forecasting)\"\n\n## Baseline code with advFE","metadata":{"papermill":{"duration":0.025818,"end_time":"2022-01-05T00:26:31.467962","exception":false,"start_time":"2022-01-05T00:26:31.442144","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Acknowledgements:\n* the main code - the notebook [💲💲G-Research- Starter LGBM Pipeline](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline)\n* the model tuning - the notebook [G-Research: XGBoost with GPU (Fit in 1min)](https://www.kaggle.com/yamqwe/g-research-xgboost-with-gpu-fit-in-1min)\n* FE - the notebook [GResearch Simple LGB Starter](https://www.kaggle.com/code1110/gresearch-simple-lgb-starter)\n* remove inf, NaN - [[Crypto] Beginner's Try for simple LGBM (En/Jp)](https://www.kaggle.com/junjitakeshima/crypto-beginner-s-try-for-simple-lgbm-en-jp)","metadata":{"papermill":{"duration":0.023501,"end_time":"2022-01-05T00:26:31.515493","exception":false,"start_time":"2022-01-05T00:26:31.491992","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0.1\"></a>\n\n## Table of Contents\n\n1. [Import libraries](#1)\n1. [Download datasets](#2)\n1. [FE](#3)\n1. [Model training](#4)\n1. [Prediction and submission](#5)","metadata":{"papermill":{"duration":0.02337,"end_time":"2022-01-05T00:26:31.562419","exception":false,"start_time":"2022-01-05T00:26:31.539049","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. Import libraries <a class=\"anchor\" id=\"1\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.02353,"end_time":"2022-01-05T00:26:31.609677","exception":false,"start_time":"2022-01-05T00:26:31.586147","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom lightgbm import LGBMRegressor\n\nimport gresearch_crypto\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":3.725834,"end_time":"2022-01-05T00:26:53.203077","exception":false,"start_time":"2022-01-05T00:26:49.477243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-21T18:17:06.807946Z","iopub.execute_input":"2022-01-21T18:17:06.808242Z","iopub.status.idle":"2022-01-21T18:17:06.813452Z","shell.execute_reply.started":"2022-01-21T18:17:06.808209Z","shell.execute_reply":"2022-01-21T18:17:06.812829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nfix_all_seeds(42)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T18:17:06.815104Z","iopub.execute_input":"2022-01-21T18:17:06.815335Z","iopub.status.idle":"2022-01-21T18:17:06.833970Z","shell.execute_reply.started":"2022-01-21T18:17:06.815307Z","shell.execute_reply":"2022-01-21T18:17:06.833227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Download datasets <a class=\"anchor\" id=\"2\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.041177,"end_time":"2022-01-05T00:26:53.286842","exception":false,"start_time":"2022-01-05T00:26:53.245665","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T18:17:06.834947Z","iopub.execute_input":"2022-01-21T18:17:06.835487Z","iopub.status.idle":"2022-01-21T18:17:19.138374Z","shell.execute_reply.started":"2022-01-21T18:17:06.835457Z","shell.execute_reply":"2022-01-21T18:17:19.137255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-21T18:17:19.139462Z","iopub.status.idle":"2022-01-21T18:17:19.139820Z","shell.execute_reply.started":"2022-01-21T18:17:19.139643Z","shell.execute_reply":"2022-01-21T18:17:19.139667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. FE <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.047191,"end_time":"2022-01-05T00:26:54.355054","exception":false,"start_time":"2022-01-05T00:26:54.307863","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def get_features(data):\n    # FE for data as row of DataFrame\n    \n    # Two new features from the competition tutorial\n    df_feat = data[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open'])\n    df_feat['Lower_Shadow'] = np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low']\n    \n    # Thanks to https://www.kaggle.com/code1110/gresearch-simple-lgb-starter\n    df_feat['high2low'] = df_feat['High'] / df_feat['Low']\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n    return df_feat","metadata":{"papermill":{"duration":0.068078,"end_time":"2022-01-05T00:26:54.470446","exception":false,"start_time":"2022-01-05T00:26:54.402368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-18T11:07:08.115729Z","iopub.execute_input":"2022-01-18T11:07:08.116071Z","iopub.status.idle":"2022-01-18T11:07:08.124105Z","shell.execute_reply.started":"2022-01-18T11:07:08.116026Z","shell.execute_reply":"2022-01-18T11:07:08.123094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_for_asset(df_train, asset_id):\n    # Get X and y\n    \n    df = df_train[df_train[\"Asset_ID\"] == asset_id]    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    #df_proc = df_proc[~df_proc.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    return X, y","metadata":{"execution":{"iopub.status.busy":"2022-01-18T11:07:08.125453Z","iopub.execute_input":"2022-01-18T11:07:08.126196Z","iopub.status.idle":"2022-01-18T11:07:08.136167Z","shell.execute_reply.started":"2022-01-18T11:07:08.126138Z","shell.execute_reply":"2022-01-18T11:07:08.135434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Model training and prediction <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"def model_training(X,y):\n    # Model training\n    \n    model = LGBMRegressor(n_estimators=5000,num_leaves=700,learning_rate=0.1)\n    model.fit(X, y)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-18T11:08:41.361895Z","iopub.execute_input":"2022-01-18T11:08:41.362207Z","iopub.status.idle":"2022-01-18T11:08:41.368877Z","shell.execute_reply.started":"2022-01-18T11:08:41.362172Z","shell.execute_reply":"2022-01-18T11:08:41.367794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y = get_data_for_asset(df_train, asset_id)    \n    model = model_training(X,y)\n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-18T11:08:41.370485Z","iopub.execute_input":"2022-01-18T11:08:41.370865Z","iopub.status.idle":"2022-01-18T11:10:50.70515Z","shell.execute_reply.started":"2022-01-18T11:08:41.370824Z","shell.execute_reply":"2022-01-18T11:10:50.704423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model and it's possibility for the prediction \nprint(\"Check the model and it's possibility for the prediction\")\nx = get_features(df_train.iloc[1])\ny_pred = models[0].predict([x])\ny_pred[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-18T11:18:59.270338Z","iopub.execute_input":"2022-01-18T11:18:59.270941Z","iopub.status.idle":"2022-01-18T11:18:59.353436Z","shell.execute_reply.started":"2022-01-18T11:18:59.270843Z","shell.execute_reply":"2022-01-18T11:18:59.352419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Prediction and submission <a class=\"anchor\" id=\"5\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"# Prediction and submission\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j, row in df_test.iterrows():\n        \n        try:\n            model = models[row['Asset_ID']]\n            x_test = get_features(row)\n            y_pred = model.predict([x_test])[0]\n\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        except:\n            print(f'{i}-th iteration of the test dataset, {j}-th row - there was the exception, then set Target = 0')\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n            \n        # Print just one sample row to get a feeling of what it looks like        \n        if i == 0 and j == 0:\n            print('Example of the x_test data')\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        print('Example of the prediction for test data')\n        display(df_pred)\n    df_pred['Target'] = df_pred['Target'].fillna(0)\n\n    # Send submissions\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-18T11:10:50.727295Z","iopub.execute_input":"2022-01-18T11:10:50.72754Z","iopub.status.idle":"2022-01-18T11:10:51.057957Z","shell.execute_reply.started":"2022-01-18T11:10:50.72751Z","shell.execute_reply":"2022-01-18T11:10:51.057281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I hope you find this kernel useful and enjoyable.\n\nYour comments and feedback are most welcome.","metadata":{"papermill":{"duration":0.07736,"end_time":"2022-01-05T00:27:01.043277","exception":false,"start_time":"2022-01-05T00:27:00.965917","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"[Go to Top](#0)","metadata":{"papermill":{"duration":0.076543,"end_time":"2022-01-05T00:27:01.354037","exception":false,"start_time":"2022-01-05T00:27:01.277494","status":"completed"},"tags":[]}}]}