{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Necessary Imports\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# File/data manipulation\nimport gc\nimport pathlib\nfrom tqdm.auto import tqdm\nimport joblib\nimport pathlib\nimport json\nimport glob\nimport time\nimport datetime\nfrom scipy import stats\nfrom multiprocessing import Pool, cpu_count\n\n# Visualization\nimport matplotlib.pyplot as plt\nimport matplotlib.style as style\nfrom matplotlib_venn import venn2, venn3\nimport seaborn as sns\nfrom matplotlib import pyplot\nfrom matplotlib.ticker import ScalarFormatter\nsns.set_context(\"talk\")\nstyle.use('seaborn-colorblind')\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# Model\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\n\nimport warnings\nwarnings.simplefilter('ignore')\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-17T06:13:34.685805Z","iopub.execute_input":"2021-12-17T06:13:34.686211Z","iopub.status.idle":"2021-12-17T06:13:34.710807Z","shell.execute_reply.started":"2021-12-17T06:13:34.686163Z","shell.execute_reply":"2021-12-17T06:13:34.709209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filepaths\nINPUT_DIR = '/kaggle/input/g-research-crypto-forecasting/'\nOUTPUT_DIR = './'","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:13:34.714818Z","iopub.execute_input":"2021-12-17T06:13:34.715094Z","iopub.status.idle":"2021-12-17T06:13:34.722553Z","shell.execute_reply.started":"2021-12-17T06:13:34.715060Z","shell.execute_reply":"2021-12-17T06:13:34.720782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to reduce memory usage\n# Thanks fellow Kaggle User\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n#         else:\n#             df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:13:34.725359Z","iopub.execute_input":"2021-12-17T06:13:34.726314Z","iopub.status.idle":"2021-12-17T06:13:34.745453Z","shell.execute_reply.started":"2021-12-17T06:13:34.726267Z","shell.execute_reply":"2021-12-17T06:13:34.744130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get training(and testing) data\n\n#train = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\ntrain = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv')).pipe(reduce_mem_usage)\nprint(train.shape)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:13:34.748756Z","iopub.execute_input":"2021-12-17T06:13:34.749115Z","iopub.status.idle":"2021-12-17T06:14:14.351604Z","shell.execute_reply.started":"2021-12-17T06:13:34.749054Z","shell.execute_reply":"2021-12-17T06:14:14.350540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get cryptoasset details (Real Name and Weight)\nasset_details = pd.read_csv(os.path.join(INPUT_DIR, 'asset_details.csv'))\nasset_details['Asset_ID'] = asset_details['Asset_ID'].astype(np.int8)\nprint(asset_details.shape)\nasset_details","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:14.353348Z","iopub.execute_input":"2021-12-17T06:14:14.356984Z","iopub.status.idle":"2021-12-17T06:14:14.385311Z","shell.execute_reply.started":"2021-12-17T06:14:14.356921Z","shell.execute_reply":"2021-12-17T06:14:14.384380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Look and G-Research's example_sample_submission\nexample_sample_submission = pd.read_csv(os.path.join(INPUT_DIR, 'example_sample_submission.csv'))\nprint(example_sample_submission.shape)\nexample_sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:14.386912Z","iopub.execute_input":"2021-12-17T06:14:14.387955Z","iopub.status.idle":"2021-12-17T06:14:14.406534Z","shell.execute_reply.started":"2021-12-17T06:14:14.387907Z","shell.execute_reply":"2021-12-17T06:14:14.405494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get \"test\" data. Note: just an example of the data that will be delivered by G-Research' API to test model.\n# %%time\n\n#test_df = pd.read_csv(os.path.join(INPUT_DIR, 'example_test.csv'))\ntest_df = pd.read_csv(os.path.join(INPUT_DIR, 'example_test.csv')).pipe(reduce_mem_usage)\nprint(test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:14.410813Z","iopub.execute_input":"2021-12-17T06:14:14.411140Z","iopub.status.idle":"2021-12-17T06:14:14.511715Z","shell.execute_reply.started":"2021-12-17T06:14:14.411097Z","shell.execute_reply":"2021-12-17T06:14:14.510528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Look at some of the data","metadata":{}},{"cell_type":"code","source":"# dataframe info\ntrain.info()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:14.514450Z","iopub.execute_input":"2021-12-17T06:14:14.514759Z","iopub.status.idle":"2021-12-17T06:14:14.533212Z","shell.execute_reply.started":"2021-12-17T06:14:14.514727Z","shell.execute_reply":"2021-12-17T06:14:14.531943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# missing values?\ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:14.535246Z","iopub.execute_input":"2021-12-17T06:14:14.536169Z","iopub.status.idle":"2021-12-17T06:14:15.402959Z","shell.execute_reply.started":"2021-12-17T06:14:14.536048Z","shell.execute_reply":"2021-12-17T06:14:15.402004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Total Null Target Rows = \" ,train[\"Target\"].isnull().sum())\nprint(\"Percentage of NUll rows in Training Data = {:.2f}%\".format(train[\"Target\"].isnull().sum()*100 / train.shape[0] ))","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:15.404905Z","iopub.execute_input":"2021-12-17T06:14:15.405221Z","iopub.status.idle":"2021-12-17T06:14:15.608109Z","shell.execute_reply.started":"2021-12-17T06:14:15.405176Z","shell.execute_reply":"2021-12-17T06:14:15.606947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"example_sample_submission.info()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:15.610102Z","iopub.execute_input":"2021-12-17T06:14:15.610430Z","iopub.status.idle":"2021-12-17T06:14:15.625093Z","shell.execute_reply.started":"2021-12-17T06:14:15.610382Z","shell.execute_reply":"2021-12-17T06:14:15.623858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_count= []\nfor i in range(14):\n    count = (train[\"Asset_ID\"]==i).sum()\n    asset_count.append(count)\nfig = px.bar(x = asset_details.sort_values(\"Asset_ID\")[\"Asset_Name\"],\n             y = asset_count , \n             color = asset_count ,\n             color_continuous_scale=\"Emrld\") \nfig.update_xaxes(title=\"Assets\")\nfig.update_yaxes(title = \"Number of Rows\")\nfig.update_layout(showlegend = True,\n    title = {\n        'text': 'Data Distribution ',\n        'y':0.95,\n        'x':0.5,\n        'xanchor': 'center',\n        'yanchor': 'top'})\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:15.627027Z","iopub.execute_input":"2021-12-17T06:14:15.627649Z","iopub.status.idle":"2021-12-17T06:14:16.194567Z","shell.execute_reply.started":"2021-12-17T06:14:15.627601Z","shell.execute_reply":"2021-12-17T06:14:16.192536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''fig, ax = plt.subplots(3, 5, figsize=(20, 12), sharex=True)\nax = ax.flatten()\nfor i, asset in enumerate(train['Asset_ID'].unique()):\n    train.query('Asset_ID == @asset')['Target'].hist(bins=30, color='k', alpha=0.7, ax=ax[i])\n    asset_name = asset_details.query('Asset_ID == @asset')['Asset_Name'].values[0]\n    weight = asset_details.query('Asset_ID == @asset')['Weight'].values[0]\n    ax[i].set_title(f'{asset_name}\\n(weight={weight})')\n    \nax[-1].axis('off')\nplt.tight_layout()'''","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:16.196076Z","iopub.execute_input":"2021-12-17T06:14:16.197794Z","iopub.status.idle":"2021-12-17T06:14:16.207948Z","shell.execute_reply.started":"2021-12-17T06:14:16.197732Z","shell.execute_reply":"2021-12-17T06:14:16.206615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# select train and validation period\n\n# auxiliary function, from datetime to timestamp\ntotimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%m/%d/%Y\").timetuple()))\n\ntrain_window = [totimestamp(\"01/01/2018\"), totimestamp(\"06/12/2021\")]\nvalid_window = [totimestamp(\"06/13/2021\"), totimestamp(\"09/21/2021\")]\n#train_window = [totimestamp(\"01/01/2018\"), totimestamp(\"09/21/2020\")]\n#valid_window = [totimestamp(\"09/22/2020\"), totimestamp(\"09/21/2021\")]\n\ntrain = train.set_index(\"timestamp\")\nbeg_ = train.index[0].astype('datetime64[s]')\nend_ = train.index[-1].astype('datetime64[s]')\nprint('>> data goes from ', beg_, 'to ', end_, 'shape=', train.shape)\n\n# drop rows without target\ntrain.dropna(subset=['Target'], inplace=True)\n\n# add train flag\ntrain['train_flg'] = 1\ntrain.loc[valid_window[0]:valid_window[1], 'train_flg'] = 0","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:16.209711Z","iopub.execute_input":"2021-12-17T06:14:16.210771Z","iopub.status.idle":"2021-12-17T06:14:19.837049Z","shell.execute_reply.started":"2021-12-17T06:14:16.210721Z","shell.execute_reply":"2021-12-17T06:14:19.835968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_asset_details(train, asset_details):\n    \"\"\"Add asset details to train df\n    \"\"\"\n    return train.merge(\n        asset_details\n        , how='left'\n        , on='Asset_ID'\n    )\n\n# merge asset_details\ntrain = add_asset_details(train, asset_details)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:19.841420Z","iopub.execute_input":"2021-12-17T06:14:19.841761Z","iopub.status.idle":"2021-12-17T06:14:25.544992Z","shell.execute_reply.started":"2021-12-17T06:14:19.841725Z","shell.execute_reply":"2021-12-17T06:14:25.543844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_row_feats(df):\n    df['upper_shadow'] = df['High'] / df[['Close', 'Open']].max(axis=1)\n    df['lower_shadow'] = df[['Close', 'Open']].min(axis=1) / df['Low']\n    df['open2close'] = df['Close'] / df['Open']\n    df['high2low'] = df['High'] / df['Low']\n    mean_price = df[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    median_price = df[['Open', 'High', 'Low', 'Close']].median(axis=1)\n    df['high2mean'] = df['High'] / mean_price\n    df['low2mean'] = df['Low'] / mean_price\n    df['high2median'] = df['High'] / median_price\n    df['low2median'] = df['Low'] / median_price\n    df['volume2count'] = df['Volume'] / (df['Count'] + 1)\n    df[\"opensubclose\"] = df[\"Open\"] - df[\"Close\"]\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:25.546760Z","iopub.execute_input":"2021-12-17T06:14:25.547341Z","iopub.status.idle":"2021-12-17T06:14:25.560438Z","shell.execute_reply.started":"2021-12-17T06:14:25.547294Z","shell.execute_reply":"2021-12-17T06:14:25.559213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n# feature engineering\nfeature_df = get_row_feats(train)\n\nprint(feature_df.shape)\nfeature_df.tail()","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:25.566165Z","iopub.execute_input":"2021-12-17T06:14:25.567112Z","iopub.status.idle":"2021-12-17T06:14:52.155125Z","shell.execute_reply.started":"2021-12-17T06:14:25.567022Z","shell.execute_reply":"2021-12-17T06:14:52.154110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = 'Target'\ndrops = ['timestamp', 'Asset_Name', 'Weight', 'train_flg', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']\nfeatures = [f for f in train.columns if f not in drops + [target]]\ncategoricals = ['Asset_ID']\n\nprint('{:,} features: {}'.format(len(features), features))","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:52.157638Z","iopub.execute_input":"2021-12-17T06:14:52.158307Z","iopub.status.idle":"2021-12-17T06:14:52.166756Z","shell.execute_reply.started":"2021-12-17T06:14:52.158241Z","shell.execute_reply":"2021-12-17T06:14:52.165668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train (full model)\nmodel = LGBMRegressor(#n_estimators=101,\n                      n_estimators=10000,\n                      objective='regression',\n                      metric='rmse',\n                      boosting_type='gbdt',\n                      max_depth=-1,\n                      learning_rate=0.01,\n                      subsample=0.72,\n                      subsample_freq=4,\n                      feature_fraction=0.4,\n                      bagging_fraction=0.4,\n                      lambda_l1=1,\n                      lambda_l2=1,\n                      seed=46,)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:52.168586Z","iopub.execute_input":"2021-12-17T06:14:52.169215Z","iopub.status.idle":"2021-12-17T06:14:52.181116Z","shell.execute_reply.started":"2021-12-17T06:14:52.169163Z","shell.execute_reply":"2021-12-17T06:14:52.180058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(feature_df.query('train_flg == 1')[features],\n                    feature_df.query('train_flg == 1')[target].values,\n                    eval_set=[(feature_df.query('train_flg == 0')[features],\n                               feature_df.query('train_flg == 0')[target].values)],\n                    verbose=-1,\n                    early_stopping_rounds=100,\n                    categorical_feature=categoricals,)\n\n# save model\njoblib.dump(model, os.path.join(OUTPUT_DIR, 'lgb_model_val.pkl'))\nprint('lgb model saved!')\n\n# feature importance\nfi_df = pd.DataFrame()\nfi_df['features'] = features\nfi_df['importance'] = model.booster_.feature_importance(importance_type=\"gain\")","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:14:52.182837Z","iopub.execute_input":"2021-12-17T06:14:52.183446Z","iopub.status.idle":"2021-12-17T06:22:02.139009Z","shell.execute_reply.started":"2021-12-17T06:14:52.183368Z","shell.execute_reply":"2021-12-17T06:22:02.137938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot feature importance\nfig, ax = plt.subplots(1, 1, figsize=(7, 15))\nsns.barplot(\n    x='importance'\n    , y='features'\n    , data=fi_df.sort_values(by=['importance'], ascending=False)\n    , ax=ax\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:22:02.141852Z","iopub.execute_input":"2021-12-17T06:22:02.142477Z","iopub.status.idle":"2021-12-17T06:22:02.493376Z","shell.execute_reply.started":"2021-12-17T06:22:02.142416Z","shell.execute_reply":"2021-12-17T06:22:02.492402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/291845\n\ndef weighted_correlation(a, b, weights):\n\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / np.sqrt(var_a * var_b)\n\n    return corr\n\n# Evaulating the model by computing weighted correlation\nmodel = joblib.load(os.path.join(OUTPUT_DIR, 'lgb_model_val.pkl'))\nval_df = train.query('train_flg == 0').copy()\nval_df['Prediction'] = model.predict(val_df[features])\nfor asset in val_df['Asset_ID'].unique():\n    tmp = val_df.query('Asset_ID == @asset')\n    coin = tmp['Asset_Name'].values[0]\n    corr = weighted_correlation(tmp['Prediction'], tmp['Target'], tmp['Weight'])\n    print('')\n    print('- {}: Validation Score (weighted correlation) = {:.4f}'.format(coin, corr))\n\ncorr = weighted_correlation(val_df['Prediction'], val_df['Target'], val_df['Weight'])\nprint('=> Overall Validation Score (weighted correlation) = {:.4f}'.format(corr))","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:22:02.494970Z","iopub.execute_input":"2021-12-17T06:22:02.495511Z","iopub.status.idle":"2021-12-17T06:22:26.172167Z","shell.execute_reply.started":"2021-12-17T06:22:02.495444Z","shell.execute_reply":"2021-12-17T06:22:26.170493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()   # initialize the environment\niter_test = env.iter_test()    # an iterator which loops over the test set and sample submission\nfor (test_df, sample_prediction_df) in iter_test:\n    # feature engineering\n    test_df = get_row_feats(test_df)\n    \n    # inference\n    sample_prediction_df['Target'] = model.predict(test_df[features])  # make your predictions here\n    \n    # register your predictions\n    env.predict(sample_prediction_df)","metadata":{"execution":{"iopub.status.busy":"2021-12-17T06:22:26.173764Z","iopub.execute_input":"2021-12-17T06:22:26.174419Z","iopub.status.idle":"2021-12-17T06:22:26.262227Z","shell.execute_reply.started":"2021-12-17T06:22:26.174369Z","shell.execute_reply":"2021-12-17T06:22:26.260503Z"},"trusted":true},"execution_count":null,"outputs":[]}]}