{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gresearch_crypto\nimport pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport traceback","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:48:11.891725Z","iopub.execute_input":"2022-01-07T18:48:11.892475Z","iopub.status.idle":"2022-01-07T18:48:13.984451Z","shell.execute_reply.started":"2022-01-07T18:48:11.89234Z","shell.execute_reply":"2022-01-07T18:48:13.983414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = \"/kaggle/input/g-research-crypto-forecasting/\"\ndf_train = pd.read_csv(path + \"train.csv\")\ndf_test = pd.read_csv(path + \"example_test.csv\")\ndf_asset_details = pd.read_csv(path + \"asset_details.csv\").set_index(\"Asset_ID\")\ndf_supp_train = pd.read_csv(path + \"supplemental_train.csv\")\nSEED = 137\n\nenv = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:48:13.985907Z","iopub.execute_input":"2022-01-07T18:48:13.986149Z","iopub.status.idle":"2022-01-07T18:49:19.657437Z","shell.execute_reply.started":"2022-01-07T18:48:13.986119Z","shell.execute_reply":"2022-01-07T18:49:19.65669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:49:19.658909Z","iopub.execute_input":"2022-01-07T18:49:19.659824Z","iopub.status.idle":"2022-01-07T18:49:19.676844Z","shell.execute_reply.started":"2022-01-07T18:49:19.659777Z","shell.execute_reply":"2022-01-07T18:49:19.676176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"used_columns = ['Count', 'Open', 'High', 'Low', 'Close',\n                'Volume', 'VWAP', 'upper_shadow', 'lower_shadow', 'Close/Open',\n                'hlco_ratio', 'spread', 'mean_trade', 'log_price_change', 'high_div_low',\n                'trade', 'gtrade', 'shadow1', 'shadow3', 'shadow5', 'upper_shadow_log',\n                'lower_shadow_log']\n\nparams = {\n    \"n_estimators\": 5000,\n    \"num_leaves\" : 300,\n    \"learning_rate\" : 0.09,\n    \"random_seed\" : SEED,\n    \"n_jobs\": 4\n}\n\ndef enrichment_features(df: pd.DataFrame) -> pd.DataFrame:\n    df['upper_shadow'] = df['High'] - np.maximum(df['Close'], df['Open'])\n    df['lower_shadow'] = np.minimum(df['Close'], df['Open']) - df['Low']\n    df[\"Close/Open\"] = df[\"Close\"] / df[\"Open\"]\n    df['hlco_ratio'] = (df['High'] - df['Low'])/(df['Close']-df['Open'])\n    df['spread'] = df['High'] - df['Low']\n    df['mean_trade'] = df['Volume']/df['Count']\n    df['log_price_change'] = np.log(df['Close']/df['Open'])\n    df[\"high_div_low\"] = df[\"High\"] / df[\"Low\"]\n    df['trade']=df['Close']-df['Open']\n    df['gtrade']=df['trade']/df['Count']\n    df['shadow1']=df['trade']/df['Volume']\n    df['shadow3']=df['upper_shadow']/df['Volume']\n    df['shadow5']=df['lower_shadow']/df['Volume']\n    df['upper_shadow_log']=np.log(df['upper_shadow'])\n    df['lower_shadow_log']=np.log(df['lower_shadow'])\n\n    return df[used_columns]","metadata":{"execution":{"iopub.status.busy":"2022-01-07T19:11:50.921096Z","iopub.execute_input":"2022-01-07T19:11:50.921469Z","iopub.status.idle":"2022-01-07T19:11:50.937788Z","shell.execute_reply.started":"2022-01-07T19:11:50.921432Z","shell.execute_reply":"2022-01-07T19:11:50.936769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Validation test","metadata":{}},{"cell_type":"code","source":"# from scipy.stats import pearsonr\n\n# def log(model,X_train, X_valid, y_train, y_valid,train_split=1.0):\n#     if train_split > 0:\n#         X_train=X_train[:int(train_split*X_train.shape[0])]\n#         y_train=y_train[:int(train_split*y_train.shape[0])]\n    \n#         pred=model.predict(X_train)\n#         print('Training :- ')\n#         print(f'MSE : {np.mean((y_train-pred)**2)}')\n#         print(f'CV : {pearsonr(pred,y_train)[0]}')\n#     pred=model.predict(X_valid)\n#     print('Validation :- ')\n#     print(f'MSE : {np.mean((y_valid-pred)**2)}')\n#     print(f'CV : {pearsonr(pred,y_valid)[0]}')","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:59:46.395657Z","iopub.execute_input":"2022-01-07T18:59:46.396423Z","iopub.status.idle":"2022-01-07T18:59:46.404088Z","shell.execute_reply.started":"2022-01-07T18:59:46.396366Z","shell.execute_reply":"2022-01-07T18:59:46.403288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def validation(df_train, asset_id):\n#     asset_name = df_asset_details.iloc[asset_id][\"Asset_Name\"]\n#     df = df_train[df_train[\"Asset_ID\"] == asset_id]\n   \n#     currency_df = df_train[df_train[\"Asset_ID\"] == asset_id]\n#     currency_df = currency_df.dropna()\n#     currency_y = currency_df[\"Target\"]\n#     currency_x = enrichment_features(currency_df)\n    \n#     X_train=currency_x[:int(0.7*currency_x.shape[0])]\n#     y_train=currency_y[:int(0.7*currency_y.shape[0])]\n#     X_test=currency_x[int(0.7*currency_x.shape[0]):]\n#     y_test=currency_y[int(0.7*currency_x.shape[0]):]\n\n#     model = LGBMRegressor(**params)\n#     model.fit(X_train, y_train)\n#     print(f'[Finished Training] evaluating for {asset_name}')\n#     log(model,X_train, X_test, y_train, y_test,0.3)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:59:49.414088Z","iopub.execute_input":"2022-01-07T18:59:49.414756Z","iopub.status.idle":"2022-01-07T18:59:49.423813Z","shell.execute_reply.started":"2022-01-07T18:59:49.41471Z","shell.execute_reply":"2022-01-07T18:59:49.423004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# validation(df_train, 2)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T19:12:06.325111Z","iopub.execute_input":"2022-01-07T19:12:06.32574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1000\n\n[Finished Training] evaluating for Bitcoin\nTraining :- \nMSE : 1.8713016691819038e-05\nCV : 0.47574778046270555\nValidation :- \nMSE : 1.7527629595927818e-05\nCV : 0.004341025230372545\n\n3000\n\n[Finished Training] evaluating for Bitcoin\nTraining :- \nMSE : 1.266277321245002e-05\nCV : 0.7357600141817864\nValidation :- \nMSE : 1.9788559920537545e-05\nCV : 0.002839122533392276\n\n5000\n\n[Finished Training] evaluating for Bitcoin\nTraining :- \nMSE : 7.1893931407124685e-06\nCV : 0.8870647218100292\nValidation :- \nMSE : 1.890073088293228e-05\nCV : 0.0017710093865425422","metadata":{}},{"cell_type":"markdown","source":"Train and submit","metadata":{}},{"cell_type":"code","source":"def init_model (x, y):\n    model = LGBMRegressor(**params)\n    model.fit(x, y)\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:49:21.198114Z","iopub.status.idle":"2022-01-07T18:49:21.198477Z","shell.execute_reply.started":"2022-01-07T18:49:21.198273Z","shell.execute_reply":"2022-01-07T18:49:21.19829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\n\nfor asset_id in range(df_asset_details.shape[0]):\n    asset_name = df_asset_details.iloc[asset_id][\"Asset_Name\"]\n    currency_df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    currency_df = currency_df.dropna()\n    \n    currency_y = currency_df[\"Target\"]\n    currency_x = enrichment_features(currency_df)\n    \n    models[asset_id] = init_model(currency_x, currency_y)\n    \n    print(f\"{asset_name} model was initialized.\")","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:49:21.199342Z","iopub.status.idle":"2022-01-07T18:49:21.199653Z","shell.execute_reply.started":"2022-01-07T18:49:21.199497Z","shell.execute_reply":"2022-01-07T18:49:21.199513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    for j , row in df_test.iterrows():        \n        if models[row['Asset_ID']] is not None:\n            try:\n                model = models[row['Asset_ID']]\n                x_test = enrichment_features(row)\n                y_pred = model.predict(pd.DataFrame([x_test]))[0]\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n            except:\n                df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n                traceback.print_exc()\n        else: \n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0  \n    \n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-07T18:49:21.201061Z","iopub.status.idle":"2022-01-07T18:49:21.201371Z","shell.execute_reply.started":"2022-01-07T18:49:21.201204Z","shell.execute_reply":"2022-01-07T18:49:21.201219Z"},"trusted":true},"execution_count":null,"outputs":[]}]}