{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gresearch_crypto\nimport matplotlib.pyplot as plt\nenv = gresearch_crypto.make_env()\nfrom datetime import datetime\nimport gc\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom lightgbm import LGBMRegressor\nfrom sklearn.model_selection import StratifiedKFold,KFold\nfrom sklearn.model_selection import train_test_split\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-12T08:17:56.174382Z","iopub.execute_input":"2022-01-12T08:17:56.174761Z","iopub.status.idle":"2022-01-12T08:17:58.288078Z","shell.execute_reply.started":"2022-01-12T08:17:56.174654Z","shell.execute_reply":"2022-01-12T08:17:58.287045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model build and submission\n\nThe steps for data creation can be found here: \n\n[https://www.kaggle.com/craniket/g-research-data-creation](http://).\n\nHere I build a basic LGBM model and submit the predictions.","metadata":{}},{"cell_type":"code","source":"data_dir1 = \"/kaggle/input/training/\"\nx_train = pd.read_csv(data_dir1+\"x_train.csv\")\ny_train = pd.read_csv(data_dir1+\"y_train.csv\")\ny_test = pd.read_csv(data_dir1+\"y_test.csv\")\nweight_train = pd.read_csv(data_dir1+\"weight_train.csv\")\nweight_test = pd.read_csv(data_dir1+\"weight_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:17:58.289844Z","iopub.execute_input":"2022-01-12T08:17:58.290105Z","iopub.status.idle":"2022-01-12T08:19:20.063429Z","shell.execute_reply.started":"2022-01-12T08:17:58.290075Z","shell.execute_reply":"2022-01-12T08:19:20.062478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir2 = \"/kaggle/input/testxx/\"\nx_test = pd.read_csv(data_dir2+\"x_test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:19:20.064602Z","iopub.execute_input":"2022-01-12T08:19:20.065066Z","iopub.status.idle":"2022-01-12T08:19:42.195718Z","shell.execute_reply.started":"2022-01-12T08:19:20.065017Z","shell.execute_reply":"2022-01-12T08:19:42.19494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dropping unnecessary columns","metadata":{}},{"cell_type":"code","source":"x_train = x_train.drop([\"timestamp\",\"filt\"],axis = 1)\nx_test = x_test.drop([\"timestamp\",\"filt\"],axis = 1)\ny_train = y_train.drop([\"timestamp\"],axis = 1)\ny_test = y_test.drop([\"timestamp\"],axis = 1)\nx_train['Mean'] = x_train[['Open', 'High', 'Low', 'Close']].mean(axis=1)    \nx_train['High/Mean'] = x_train['High'] / x_train['Mean']\nx_train['Low/Mean'] = x_train['Low'] / x_train['Mean']\nx_train['Volume/Count'] = x_train['Volume'] / (x_train['Count'] + 1)\nweight_train = weight_train.drop([\"timestamp\"],axis = 1).values\nweight_test = weight_test.drop([\"timestamp\"],axis = 1).values","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:19:42.198153Z","iopub.execute_input":"2022-01-12T08:19:42.198541Z","iopub.status.idle":"2022-01-12T08:19:44.520008Z","shell.execute_reply.started":"2022-01-12T08:19:42.198496Z","shell.execute_reply":"2022-01-12T08:19:44.519153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:19:44.528404Z","iopub.execute_input":"2022-01-12T08:19:44.530181Z","iopub.status.idle":"2022-01-12T08:19:44.538888Z","shell.execute_reply.started":"2022-01-12T08:19:44.530133Z","shell.execute_reply":"2022-01-12T08:19:44.538049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here I define the metrics as per rules, but in practice implementing this on test data took up too much space.","metadata":{}},{"cell_type":"code","source":"def wmean(x, w):\n    return np.sum(x * w) / np.sum(w)\n\ndef wcov(x, y, w):\n    return np.sum(w * (x - wmean(x, w)) * (y - wmean(y, w))) / np.sum(w)\n\ndef wcorr(x, y, w):\n    return wcov(x, y, w) / np.sqrt(wcov(x, x, w) * wcov(y, y, w))\n\ndef eval_wcorr(preds, dataset):\n    y =dataset.get_label()    \n    w = dataset.get_weight()\n    return 'eval_wcorr', wcorr(preds, y, w), True\n\nparams = {'n_estimators': 500,\n        'objective': 'regression_l1',  'metric': 'None',\n        'boosting_type': 'gbdt',\n        'max_depth': -1,\n        'learning_rate': 0.001,\n        'subsample': 0.4,\n        'subsample_freq': 4,\n        'feature_fraction': 0.4,\n        'lambda_l1': 1,\n        'lambda_l2': 1,\n        'seed': 123,\n        'verbose': -1,\n        }\npred = []\nscore=[]\nevals_result = {}\nmodels = []\ntest_dataset = lgb.Dataset(x_test, y_test,weight = weight_test)\nfor idx_t,idx_v in KFold(n_splits = 3, shuffle = True, random_state = 2).split(x_train):\n    X_t,y_t,X_v,y_v = x_train.iloc[idx_t],y_train.iloc[idx_t],x_train.iloc[idx_v],y_train.iloc[idx_v]\n    weight_t,weight_v = weight_train[idx_t],weight_train[idx_v]\n    train_dataset = lgb.Dataset(X_t, y_t,weight = weight_t)\n    val_dataset = lgb.Dataset(X_v, y_v,weight =weight_v)    \n    model = lgb.train(params,\n                          #early_stopping_rounds=1000,\n                          verbose_eval = 100,\n                          feval=eval_wcorr,\n                          train_set = train_dataset, \n                          valid_sets = [val_dataset],\n                          evals_result = evals_result \n                         )\n    models.append(model)\n    #preds2 = model.predict(x_test)\n    #sc = eval_wcorr(preds2, test_dataset)\n    #score = score.append(sc)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:19:44.540821Z","iopub.execute_input":"2022-01-12T08:19:44.541148Z","iopub.status.idle":"2022-01-12T08:26:04.126875Z","shell.execute_reply.started":"2022-01-12T08:19:44.541106Z","shell.execute_reply":"2022-01-12T08:26:04.125913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#preds2.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:26:04.129495Z","iopub.execute_input":"2022-01-12T08:26:04.130139Z","iopub.status.idle":"2022-01-12T08:26:04.135771Z","shell.execute_reply.started":"2022-01-12T08:26:04.130091Z","shell.execute_reply":"2022-01-12T08:26:04.134512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del x_train\ndel x_test\ndel train_dataset\ndel test_dataset\ndel val_dataset\ndel y_train\ndel y_test\nimport gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:26:04.137161Z","iopub.execute_input":"2022-01-12T08:26:04.137471Z","iopub.status.idle":"2022-01-12T08:26:04.330641Z","shell.execute_reply.started":"2022-01-12T08:26:04.137439Z","shell.execute_reply":"2022-01-12T08:26:04.329789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test()# an iterator which loops over the test set and sample submission\nfor (test_df, sample_prediction_df) in iter_test:\n    test_df['change'] = test_df['Close']/test_df['Open']\n    test_df['variation'] = test_df['High']/test_df['Low']\n    test_df['change_diff'] = test_df['Close'] - test_df['Open']\n    test_df['variation_diff'] = test_df['High'] - test_df['Low']\n    def hlco_ratio(df): return (df['High'] - df['Low'])/(df['Close']-df['Open'])\n    def upper_shadow(df): return df['High'] - np.maximum(df['Close'], df['Open'])\n    def lower_shadow(df): return np.minimum(df['Close'], df['Open']) - df['Low']\n    test_df['Upper_Shadow'] = upper_shadow(test_df)\n    test_df['Lower_Shadow'] = lower_shadow(test_df)\n    test_df['hlco_ration'] = hlco_ratio(test_df)\n    test_df['Mean'] = test_df[['Open', 'High', 'Low', 'Close']].mean(axis=1)    \n    test_df['High/Mean'] = test_df['High'] / test_df['Mean']\n    test_df['Low/Mean'] = test_df['Low'] / test_df['Mean']\n    test_df['Volume/Count'] = test_df['Volume'] / (test_df['Count'] + 1)\n\n    preds2 = []\n    for i in range(3):\n        model = models[i]\n        preds = model.predict(test_df.drop([\"timestamp\",\"row_id\"],axis=1))\n        preds2.append(preds)\n    sample_prediction_df['Target'] = np.mean(np.column_stack(preds2),axis=1)  # make your predictions here\n    env.predict(sample_prediction_df) ","metadata":{"execution":{"iopub.status.busy":"2022-01-12T08:26:04.333073Z","iopub.execute_input":"2022-01-12T08:26:04.333969Z","iopub.status.idle":"2022-01-12T08:26:04.513475Z","shell.execute_reply.started":"2022-01-12T08:26:04.333912Z","shell.execute_reply":"2022-01-12T08:26:04.512141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}