{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0\"></a>\n# Competition \"[G-Research Crypto Forecasting](https://www.kaggle.com/c/g-research-crypto-forecasting)\"\n\n## Baseline code with advFE","metadata":{"papermill":{"duration":0.025818,"end_time":"2022-01-05T00:26:31.467962","exception":false,"start_time":"2022-01-05T00:26:31.442144","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Acknowledgements:\n* the main code - the notebook [💲💲G-Research- Starter LGBM Pipeline](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline)\n* the model tuning - the notebook [G-Research: XGBoost with GPU (Fit in 1min)](https://www.kaggle.com/yamqwe/g-research-xgboost-with-gpu-fit-in-1min)\n* FE - the notebook [GResearch Simple LGB Starter](https://www.kaggle.com/code1110/gresearch-simple-lgb-starter)\n* remove inf, NaN - [[Crypto] Beginner's Try for simple LGBM (En/Jp)](https://www.kaggle.com/junjitakeshima/crypto-beginner-s-try-for-simple-lgbm-en-jp)","metadata":{"papermill":{"duration":0.023501,"end_time":"2022-01-05T00:26:31.515493","exception":false,"start_time":"2022-01-05T00:26:31.491992","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"<a class=\"anchor\" id=\"0.1\"></a>\n\n## Table of Contents\n\n1. [Import libraries](#1)\n1. [Download datasets](#2)\n1. [FE](#3)\n1. [Model training](#4)\n1. [Prediction and submission](#5)","metadata":{"papermill":{"duration":0.02337,"end_time":"2022-01-05T00:26:31.562419","exception":false,"start_time":"2022-01-05T00:26:31.539049","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 1. Import libraries <a class=\"anchor\" id=\"1\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.02353,"end_time":"2022-01-05T00:26:31.609677","exception":false,"start_time":"2022-01-05T00:26:31.586147","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\n\n\nimport gresearch_crypto\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"papermill":{"duration":3.725834,"end_time":"2022-01-05T00:26:53.203077","exception":false,"start_time":"2022-01-05T00:26:49.477243","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-22T21:14:16.969189Z","iopub.execute_input":"2022-01-22T21:14:16.969601Z","iopub.status.idle":"2022-01-22T21:14:19.032303Z","shell.execute_reply.started":"2022-01-22T21:14:16.969499Z","shell.execute_reply":"2022-01-22T21:14:19.031448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nfix_all_seeds(42)","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:14:19.033940Z","iopub.execute_input":"2022-01-22T21:14:19.034343Z","iopub.status.idle":"2022-01-22T21:14:19.039379Z","shell.execute_reply.started":"2022-01-22T21:14:19.034311Z","shell.execute_reply":"2022-01-22T21:14:19.038403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Download datasets <a class=\"anchor\" id=\"2\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.041177,"end_time":"2022-01-05T00:26:53.286842","exception":false,"start_time":"2022-01-05T00:26:53.245665","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:14:19.040723Z","iopub.execute_input":"2022-01-22T21:14:19.040985Z","iopub.status.idle":"2022-01-22T21:15:23.030137Z","shell.execute_reply.started":"2022-01-22T21:14:19.040947Z","shell.execute_reply":"2022-01-22T21:15:23.029147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv')\ndf_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:15:23.031238Z","iopub.execute_input":"2022-01-22T21:15:23.031451Z","iopub.status.idle":"2022-01-22T21:15:23.049635Z","shell.execute_reply.started":"2022-01-22T21:15:23.031424Z","shell.execute_reply":"2022-01-22T21:15:23.048763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### EDA","metadata":{}},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:15:23.051748Z","iopub.execute_input":"2022-01-22T21:15:23.051972Z","iopub.status.idle":"2022-01-22T21:15:33.575957Z","shell.execute_reply.started":"2022-01-22T21:15:23.051943Z","shell.execute_reply":"2022-01-22T21:15:33.574676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:15:33.578566Z","iopub.execute_input":"2022-01-22T21:15:33.579696Z","iopub.status.idle":"2022-01-22T21:15:33.599390Z","shell.execute_reply.started":"2022-01-22T21:15:33.579621Z","shell.execute_reply":"2022-01-22T21:15:33.598622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# missing values\ndf_train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:15:33.600843Z","iopub.execute_input":"2022-01-22T21:15:33.601635Z","iopub.status.idle":"2022-01-22T21:15:34.121062Z","shell.execute_reply.started":"2022-01-22T21:15:33.601585Z","shell.execute_reply":"2022-01-22T21:15:34.119967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.to_datetime(df_train.timestamp[0], u)\npd.to_datetime(df_train.timestamp[100000].astype('datetime64[s]'))","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:17:30.216090Z","iopub.execute_input":"2022-01-22T21:17:30.216790Z","iopub.status.idle":"2022-01-22T21:17:30.231228Z","shell.execute_reply.started":"2022-01-22T21:17:30.216737Z","shell.execute_reply":"2022-01-22T21:17:30.230477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. FE <a class=\"anchor\" id=\"3\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{"papermill":{"duration":0.047191,"end_time":"2022-01-05T00:26:54.355054","exception":false,"start_time":"2022-01-05T00:26:54.307863","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\ndef get_features(data):\n    # FE for data as row of DataFrame\n    \n    # Two new features from the competition tutorial\n    df_feat = data[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = df_feat['High'] - np.maximum(df_feat['Close'], df_feat['Open'])\n    df_feat['Lower_Shadow'] = np.minimum(df_feat['Close'], df_feat['Open']) - df_feat['Low']\n    \n    # Thanks to https://www.kaggle.com/code1110/gresearch-simple-lgb-starter\n    df_feat['high2low'] = df_feat['High'] / df_feat['Low']\n    df_feat['volume2count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n    \n    #my features\n    df_feat['return'] = (df_feat['Close'] - df_feat['Open']) / df_feat['Open']\n#     df_feat['std_close'] = df_feat['Close'].rolling(win).std()\n#     df_feat['mean_close'] = df_feat['Close'].rolling(win).mean()\n    df_feat['close2open'] = (df_feat['Close'] - df_feat['Open']) / 2\n    df_feat['ohlc'] = (df_feat['High'] + df_feat['Open'] + df_feat['Low'] + df_feat['Close']) / 4\n    \n#     df_feat['log_return'] = np.log(df_feat['Close'] / df_feat['Close'].shift(1))\n#     df_feat['close_up_down'] = (df_feat['Close'].pct_change() > 0).astype('int')\n    \n    return df_feat","metadata":{"papermill":{"duration":0.068078,"end_time":"2022-01-05T00:26:54.470446","exception":false,"start_time":"2022-01-05T00:26:54.402368","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-01-22T21:17:38.406781Z","iopub.execute_input":"2022-01-22T21:17:38.407294Z","iopub.status.idle":"2022-01-22T21:17:38.416030Z","shell.execute_reply.started":"2022-01-22T21:17:38.407239Z","shell.execute_reply":"2022-01-22T21:17:38.415146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data_for_asset(df_train, asset_id):\n    # Get X and y\n    \n    df = df_train[df_train[\"Asset_ID\"] == asset_id]    \n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    #df_proc = df_proc[~df_proc.isin([np.nan, np.inf, -np.inf]).any(1)].reset_index(drop=True)\n    df_proc = df_proc.dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    \n    return X, y","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:17:42.231380Z","iopub.execute_input":"2022-01-22T21:17:42.232298Z","iopub.status.idle":"2022-01-22T21:17:42.239000Z","shell.execute_reply.started":"2022-01-22T21:17:42.232251Z","shell.execute_reply":"2022-01-22T21:17:42.238383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Model training and prediction <a class=\"anchor\" id=\"4\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"def model_training(X,y):\n    # Model training\n    \n    model = LGBMRegressor(n_estimators=2500, num_leaves=500, learning_rate=0.1)\n#     model = CatBoostRegressor(task_type=\"GPU\", iterations=400, learning_rate=0.02, depth=5, random_seed=42, verbose = 0)\n    model.fit(X, y)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:23:27.617125Z","iopub.execute_input":"2022-01-22T21:23:27.617495Z","iopub.status.idle":"2022-01-22T21:23:27.623607Z","shell.execute_reply.started":"2022-01-22T21:23:27.617457Z","shell.execute_reply":"2022-01-22T21:23:27.622710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nXs = {}\nys = {}\nmodels = {}\n\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n    X, y = get_data_for_asset(df_train, asset_id)    \n    model = model_training(X,y)\n    y_pred = model.predict(X)\n    print(f\"r2score: {r2_score(y, y_pred)}, \\nmse: {mean_squared_error(y, y_pred)}, \\nmae: {mean_absolute_error(y, y_pred)}\")\n    Xs[asset_id], ys[asset_id], models[asset_id] = X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-22T21:32:40.246832Z","iopub.execute_input":"2022-01-22T21:32:40.247354Z","iopub.status.idle":"2022-01-22T22:50:27.387451Z","shell.execute_reply.started":"2022-01-22T21:32:40.247316Z","shell.execute_reply":"2022-01-22T22:50:27.385880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the model and it's possibility for the prediction \nprint(\"Check the model and it's possibility for the prediction\")\n#x = get_features(df_train.iloc[1])\ny_pred = models[0].predict(X[:1])\ny_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:50:27.391386Z","iopub.execute_input":"2022-01-22T22:50:27.391738Z","iopub.status.idle":"2022-01-22T22:50:27.423754Z","shell.execute_reply.started":"2022-01-22T22:50:27.391697Z","shell.execute_reply":"2022-01-22T22:50:27.422925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Prediction and submission <a class=\"anchor\" id=\"5\"></a>\n\n[Back to Table of Contents](#0.1)","metadata":{}},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:50:27.427756Z","iopub.execute_input":"2022-01-22T22:50:27.428441Z","iopub.status.idle":"2022-01-22T22:50:27.433511Z","shell.execute_reply.started":"2022-01-22T22:50:27.428401Z","shell.execute_reply":"2022-01-22T22:50:27.432825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction and submission\n# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    \n    for j, row in df_test.iterrows():\n        \n        try:\n            model = models[row['Asset_ID']]\n            x_test = get_features(row)\n            y_pred = model.predict([x_test])[0]\n\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n        \n        except:\n            print(f'{i}-th iteration of the test dataset, {j}-th row - there was the exception, then set Target = 0')\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n            \n        # Print just one sample row to get a feeling of what it looks like        \n#         if i == 0 and j == 0:\n#             print('Example of the x_test data')\n#             display(x_test)\n    \n    # Display the first prediction dataframe\n    if i == 0:\n        print('Example of the prediction for test data')\n        display(df_pred)\n    #df_pred['Target'] = df_pred['Target'].fillna(0)\n\n    # Send submissions\n    env.predict(df_pred)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-22T22:50:27.435851Z","iopub.execute_input":"2022-01-22T22:50:27.436889Z","iopub.status.idle":"2022-01-22T22:50:28.044507Z","shell.execute_reply.started":"2022-01-22T22:50:27.436838Z","shell.execute_reply":"2022-01-22T22:50:28.043816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I hope you find this kernel useful and enjoyable.\n\nYour comments and feedback are most welcome.","metadata":{"papermill":{"duration":0.07736,"end_time":"2022-01-05T00:27:01.043277","exception":false,"start_time":"2022-01-05T00:27:00.965917","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"[Go to Top](#0)","metadata":{"papermill":{"duration":0.076543,"end_time":"2022-01-05T00:27:01.354037","exception":false,"start_time":"2022-01-05T00:27:01.277494","status":"completed"},"tags":[]}}]}