{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datatable as dt # Fast data reading/writing\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-06T15:59:22.022935Z","iopub.execute_input":"2022-02-06T15:59:22.023304Z","iopub.status.idle":"2022-02-06T15:59:22.029215Z","shell.execute_reply.started":"2022-02-06T15:59:22.023269Z","shell.execute_reply":"2022-02-06T15:59:22.028095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need a newer sklearn version:","metadata":{}},{"cell_type":"code","source":"!pip install scikit-learn==1.0.1","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:22.031684Z","iopub.execute_input":"2022-02-06T15:59:22.032302Z","iopub.status.idle":"2022-02-06T15:59:31.062155Z","shell.execute_reply.started":"2022-02-06T15:59:22.032252Z","shell.execute_reply":"2022-02-06T15:59:31.060769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First, we load the asset details so that we can load the data in the order of their asset id. Filenames are based on asset names.","metadata":{}},{"cell_type":"code","source":"asset_details = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/asset_details.csv', index_col='Asset_ID')\nnames = asset_details.sort_index().Asset_Name.values","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:31.06476Z","iopub.execute_input":"2022-02-06T15:59:31.065178Z","iopub.status.idle":"2022-02-06T15:59:31.080384Z","shell.execute_reply.started":"2022-02-06T15:59:31.065122Z","shell.execute_reply":"2022-02-06T15:59:31.079276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/bitcoin.jay\").to_pandas()\ndf.drop('index', axis=1, inplace=True)\ndf.set_index('timestamp', inplace=True)\ndf[df.isnull().values]","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:31.082058Z","iopub.execute_input":"2022-02-06T15:59:31.082432Z","iopub.status.idle":"2022-02-06T15:59:32.229103Z","shell.execute_reply.started":"2022-02-06T15:59:31.082383Z","shell.execute_reply":"2022-02-06T15:59:32.228192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since all the dataframes are too big to be stored, trained and evaluated at once, we define a function to do it in one go:","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr \nfrom glob import iglob\nfrom datetime import timedelta\n\nimportances_dict = {} # Dict of feature importances\nall_preds = [] # This will be all the made predictions stored into a dataframe\nall_trues = [] # All the true target values excluding ones that we have no predictions for (the first fold)\nasset_ids = [] # List of numpy arrays filled with the asset id for each prediction, we use it to get the weights later\n\ndef cv_evaluate(asset_name, used_features, ensemble=False, global_weight=None, num_folds=5):\n    if asset_name != 'all':\n        df = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/{asset_name.lower().replace(' ', '_')}.jay\").to_pandas() # Load asset data\n    else:\n        df = []\n        for filename in iglob(\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/*.jay\"):\n            df.append(dt.fread(filename).to_pandas()) # Load asset data\n        df = pd.concat(df)\n        \n    df.drop('index', axis=1, inplace=True)\n    df.set_index('timestamp', inplace=True)\n    X, y = df.drop('Target', axis=1)[used_features], df.Target\n    \n    if ensemble:\n        all_df = []\n        num_assets = 0\n        for filename in iglob(\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/*.jay\"):\n            asset_df = dt.fread(filename).to_pandas()\n            asset_df['asset_name'] = filename.split('/')[-1].split('.')[0]\n            all_df.append(asset_df) # Load asset data\n            num_assets += 1\n        all_df = pd.concat(all_df)\n        all_df.drop('index', axis=1, inplace=True)\n        multindex = pd.MultiIndex.from_frame(all_df[['timestamp', 'asset_name']])\n        all_df.set_index(multindex, inplace=True)\n        all_df.sort_index(inplace=True)\n        X_all, y_all = all_df.drop('Target', axis=1)[used_features], all_df.Target\n        index_diff = (X.index[0] - X_all.index[0][0]).seconds\n    \n    # Get chronological folds using time series split\n    tss = TimeSeriesSplit(n_splits=num_folds, gap=15, test_size=90*24*60) # 90 days, 15 min gap\n    split_indices = list(tss.split(X)) # Split returns generator, convert to list so that we don't have to worry about emptying it\n\n    \n    params = {'lambda_l1': 0.004498875792752676, 'lambda_l2': 0.03243290696956152, 'num_leaves': 60, \n              'max_depth': 6, 'min_data_in_leaf': 2496, 'learning_rate': 0.18502752618241153, 'n_estimators': 100,\n              'boosting_type': 'goss', 'random_state': 1}\n    \n    if ensemble:\n        global_model = LGBMRegressor(**params)\n        \n    models = []\n    \n    print(f'================================={asset_name.upper()}=================================')\n    corrs = []\n    asset_trues = [] # Same as all_trues but this is for current asset\n    asset_preds = [] # Same as all_preds but this is for current asset\n    for j, (train_indices, test_indices) in enumerate(split_indices):\n        X_train, X_test = X.iloc[train_indices], X.iloc[test_indices]\n        y_train, y_test = y[train_indices], y[test_indices]\n        \n        begin_train, begin_test = X_train.index[0], X_test.index[0]\n        end_train, end_test = X_train.index[-1], X_test.index[-1]\n        \n        print(f'Fold {j+1}\\nTrain set: {begin_train} - {end_train}\\nTest set: {begin_test}-{end_test}')\n        \n        # Training the model        \n        model = LGBMRegressor(**params)\n\n        print(f'Training fold {j+1}...')\n        model.fit(X_train, y_train)\n        models.append(model)\n        \n        if ensemble:\n            X_train_all = X_all.loc[begin_train:end_train]\n            X_test_all = X_all.loc[begin_test:end_test]\n            y_train_all = y_all.loc[begin_train:end_train]\n            y_test_all = y_all.loc[begin_test:end_test]\n            global_model.fit(X_train_all, y_train_all)\n            \n        print(f'Feature importances: {list(sorted(zip(model.feature_name_, model.feature_importances_), key=lambda x: x[1], reverse=True))}')\n        \n        # Add feature importances for current asset to importances dict, add new entry if it does not exist, otherwise append to list.\n        for k in range(len(model.feature_name_)):\n            if model.feature_name_[k] not in importances_dict.keys():\n                importances_dict[model.feature_name_[k]] = [model.feature_importances_[k]]\n            else:\n                importances_dict[model.feature_name_[k]].append(model.feature_importances_[k])         \n        \n        print(f'Predicting fold {j+1}...')\n        if ensemble:\n            train_pred = global_weight * global_model.predict(X_train) + (1-global_weight) * model.predict(X_train)\n        else:\n            train_pred = model.predict(X_train)\n\n        corr, p = pearsonr(y_train, train_pred)\n        print(f'Correlation for fold {j+1}: {corr} \\t p-value: {p}')\n\n        print(f'Predicting out of fold {j+1}...')\n        if ensemble:\n            y_pred = global_weight * global_model.predict(X_test) + (1-global_weight) * model.predict(X_test)\n        else:\n            y_pred = model.predict(X_test)\n        \n        asset_preds.append(y_pred)\n        asset_trues.append(y_test)\n\n        corr, p = pearsonr(y_test, y_pred)\n        corrs.append(corr)\n        print(f'Correlation for OOF {j+1}: {corr} \\t p-value: {p}')\n      \n    print(f'\\n\\nMEAN OOF CORRELATION FOR {asset_name.upper()}: {sum(corrs)/len(corrs)}\\n\\n')\n\n#     if name != all:\n        # Concatenate the per fold predictions so that asset_preds and asset_trues contain all values for this asset.\n        # All_preds and all_trues are then updated with these values.\n#         asset_preds = np.concatenate(asset_preds)\n#         asset_trues = np.concatenate(asset_trues)\n#         all_preds.append(asset_preds)\n#         all_trues.append(asset_trues)\n#         asset_id = asset_details[asset_details['Asset_Name'] == asset_n].index[0]\n#         asset_ids.append(np.full((len(asset_preds)), asset_id))\n    \n    return models","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.232699Z","iopub.execute_input":"2022-02-06T15:59:32.233269Z","iopub.status.idle":"2022-02-06T15:59:32.267996Z","shell.execute_reply.started":"2022-02-06T15:59:32.233212Z","shell.execute_reply":"2022-02-06T15:59:32.266658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We now use the function to load and train all the models iteratively:","metadata":{}},{"cell_type":"code","source":"used_features = ['MACD_crossover_norm', 'stochastic_crossover', 'RSI', 'log_ret1', 'log_ret30', 'log_ret240', 'log_ret1440', 'mfi']\nasset_models = []\n\n#global_model = cv_evaluate('all', used_features)\n\nfor name in ['Bitcoin', 'Cardano', 'Maker', 'Tron']:\n    asset_models.append(cv_evaluate(name, used_features, ensemble=True, global_weight=0.5))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T16:08:38.798075Z","iopub.execute_input":"2022-02-06T16:08:38.798505Z","iopub.status.idle":"2022-02-06T16:54:19.201764Z","shell.execute_reply.started":"2022-02-06T16:08:38.798438Z","shell.execute_reply":"2022-02-06T16:54:19.196255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import shap\nbitcoin_last = asset_models[0][-1]\ndata = dt.fread(f\"/kaggle/input/crypto-challenge-mlii-project-feature-eng-2/bitcoin.jay\").to_pandas()[used_features].iloc[:60*24*30]\nshap_values = shap.TreeExplainer(bitcoin_last).shap_values(data)\nshap.summary_plot(shap_values, data)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-02-06T17:13:58.972299Z","iopub.execute_input":"2022-02-06T17:13:58.972676Z","iopub.status.idle":"2022-02-06T17:14:12.294712Z","shell.execute_reply.started":"2022-02-06T17:13:58.972635Z","shell.execute_reply":"2022-02-06T17:14:12.29397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cardano_last = asset_models[1][-1]\nshap_values = shap.TreeExplainer(cardano_last).shap_values(data)\nshap.summary_plot(shap_values, data)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T17:17:21.757304Z","iopub.execute_input":"2022-02-06T17:17:21.758013Z","iopub.status.idle":"2022-02-06T17:17:34.376187Z","shell.execute_reply.started":"2022-02-06T17:17:21.757968Z","shell.execute_reply":"2022-02-06T17:17:34.374937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"maker_last = asset_models[2][-1]\nshap_values = shap.TreeExplainer(maker_last).shap_values(data)\nshap.summary_plot(shap_values, data)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T17:17:34.378232Z","iopub.execute_input":"2022-02-06T17:17:34.378562Z","iopub.status.idle":"2022-02-06T17:17:46.178174Z","shell.execute_reply.started":"2022-02-06T17:17:34.378522Z","shell.execute_reply":"2022-02-06T17:17:46.177104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tron_last = asset_models[3][-1]\nshap_values = shap.TreeExplainer(tron_last).shap_values(data)\nshap.summary_plot(shap_values, data)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T17:20:33.817056Z","iopub.execute_input":"2022-02-06T17:20:33.817939Z","iopub.status.idle":"2022-02-06T17:20:33.846869Z","shell.execute_reply.started":"2022-02-06T17:20:33.817879Z","shell.execute_reply":"2022-02-06T17:20:33.845221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualizing feature imporatances with a boxplot:","metadata":{}},{"cell_type":"code","source":"importances = pd.DataFrame(importances_dict).melt(var_name='Feature', value_name='Importance')\nprint(importances)\nplt.figure(figsize=(10, 30))\nsns.boxplot(data=importances, x='Importance', y='Feature')","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.330809Z","iopub.status.idle":"2022-02-06T15:59:32.331328Z","shell.execute_reply.started":"2022-02-06T15:59:32.331101Z","shell.execute_reply":"2022-02-06T15:59:32.331131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We create a dataframe with ids and target predictions so we can map them all to weights:","metadata":{}},{"cell_type":"code","source":"eval_df = pd.DataFrame({'Id': np.hstack(asset_ids), 'True': np.hstack(all_trues), 'Prediction': np.hstack(all_preds)})\neval_df = eval_df.join(asset_details, on='Id')","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.332411Z","iopub.status.idle":"2022-02-06T15:59:32.333138Z","shell.execute_reply.started":"2022-02-06T15:59:32.332873Z","shell.execute_reply":"2022-02-06T15:59:32.332913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we define the correlation metric:","metadata":{}},{"cell_type":"code","source":"eval_df.head(800000)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.334384Z","iopub.status.idle":"2022-02-06T15:59:32.33495Z","shell.execute_reply.started":"2022-02-06T15:59:32.334719Z","shell.execute_reply":"2022-02-06T15:59:32.334753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wcorr(y_true, y_pred, weights): # Adapted from the discussion post 'Evaluation Metric Clarification'\n    sum_w = np.sum(weights)\n    mean_true = np.sum(y_true * weights) / sum_w\n    mean_pred = np.sum(y_pred * weights) / sum_w\n    var_true = np.sum(weights * np.square(y_true - mean_true)) / sum_w\n    var_pred = np.sum(weights * np.square(y_pred - mean_pred)) / sum_w\n\n    cov = np.sum((y_true * y_pred * weights)) / sum_w - mean_true * mean_pred\n    corr = cov / np.sqrt(var_true * var_pred)\n\n    return corr","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.336029Z","iopub.status.idle":"2022-02-06T15:59:32.336601Z","shell.execute_reply.started":"2022-02-06T15:59:32.336323Z","shell.execute_reply":"2022-02-06T15:59:32.336353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Calculating the metric (probably more representative with a standard training size) ","metadata":{}},{"cell_type":"code","source":"print(wcorr(eval_df['True'], eval_df['Prediction'], eval_df['Weight']))","metadata":{"execution":{"iopub.status.busy":"2022-02-06T15:59:32.337712Z","iopub.status.idle":"2022-02-06T15:59:32.338231Z","shell.execute_reply.started":"2022-02-06T15:59:32.338001Z","shell.execute_reply":"2022-02-06T15:59:32.33803Z"},"trusted":true},"execution_count":null,"outputs":[]}]}