{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==0.24.2","metadata":{"_uuid":"35d9ff38-86eb-4ce6-94e1-0fef4fc74b31","_cell_guid":"97deee1c-1a28-4ff0-abde-197857d911ca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-04T15:16:03.026848Z","iopub.execute_input":"2022-01-04T15:16:03.027365Z","iopub.status.idle":"2022-01-04T15:16:20.039815Z","shell.execute_reply.started":"2022-01-04T15:16:03.027260Z","shell.execute_reply":"2022-01-04T15:16:20.038785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Motivation\n\nI'm trying to solo this competion for less than a month. And this is the pipeline I'm gonna do.\n\n## Pipeline\n- **Data Loading from a preprocessed dataset(compressed and target reconstructed)** (https://www.kaggle.com/axzhang/fork-of-dataprepare2)\n- **Define the Cross-Validation Pipeline since LB is meaningless** (This notebook)\n- **Define features for each asset** (**TODO**)\n  - Continuous features such as technical analysis.\n  - Most features depend on number of looking-back window such as moving average. We should choose the window to use according to \n    * 1) **momentum**: the feature that has positve correlation with the target.\n    * 2) **mean-reversion**: the feature that has negative correlation with the target.\n  - Sparse features such as some break event.\n    - Bear-Bull Market indicator: This kind of indicator is too sparse for minute-level data. Better train a different model for each type of market.\n    - Trendy Occilator\n     \n- **Train a asset-specific model using GappedTimeSeries to avoid overfitting to a specific time period.** (**TODO**)\n - Each model should be weighted according to some weights (considering time and performance in validation set)\n - For some assets like **Maker**, I found that there are lots of gapped in the datasets. Currently, we skip them.\n     - Using external datasets from Binance to fill the **NaN**s\n- **Train a asset-agostic model using all the same cross-validation method.** (**TODO**)\n - Reasons 1) For minute-level data, I believe there are common patterns for all assets because of the micro-structure of the market is universally similar. \n - Reasons 2) The target is constructed using all sets. So somehow they are correlated.\n - Avoid overfitting: we should do factor-neutralization for each feature since 1) each feature has different scale and 2) we don't want to let the model overfit one asset.\n \n- **Massive Parameter Tuning** (**TODO**)\n    * Using HPO package","metadata":{}},{"cell_type":"code","source":"TEST_DAY = 3 * 30\n# train_day = 6 * 30\nTRAIN_DAY = -1\nGAP_DAY = 15\nN_SPLIT = 5\nCKPT = \"ckpt\"\nSKIPS = ['Maker', \"Monero\", \"Stellar\"]\n\nMODEL_PARAMS = {\n    \"n_estimators\": 1000,\n    \"early_stopping_round\": 50,\n    \"max_depth\": 4,  # choose a very shallow depth to ovoid overfitting.\n    \"random_seed\": 2021,\n    \"learning_rate\": 1e-3,\n    \"colsample_bytree\": 0.3,  # For the most of the time, trader only looks at <= 5 features to make decision. Accordingly, we limite the feature-wise sample size.\n    \"subsample\": 0.8,\n    \"metric\": \"regression_l2\",\n    \"verbosity\": -1,\n    \"min_data_in_leaf\": 100,\n    \"device\": \"gpu\"\n}","metadata":{"_uuid":"ec27d8e6-e75b-4f1e-84da-301455db157f","_cell_guid":"bbd9a56d-875f-4c44-a011-a1d4df4f7c48","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-04T15:27:57.943127Z","iopub.execute_input":"2022-01-04T15:27:57.943467Z","iopub.status.idle":"2022-01-04T15:27:57.969090Z","shell.execute_reply.started":"2022-01-04T15:27:57.943380Z","shell.execute_reply":"2022-01-04T15:27:57.968436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb \nimport sklearn\nimport os\nimport json\nfrom scipy.stats import pearsonr\nimport logging\n\ndef pearson_eval(preds, train_data):\n    \"\"\"customized lgb evaluation method \"\"\"\n    labels = np.nan_to_num(train_data.get_label())\n    return 'corr', pearsonr(labels, np.nan_to_num(preds))[0], True\n\nlogger = logging.getLogger()\nlogger.setLevel(logging.INFO)\nlgb.register_logger(logger)\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / (np.sqrt(var_a * var_b) + 1e-12)\n    return corr\n\ndef validate_one_symble(model, features, label):\n    pred = model.predict(features)\n    dummy_weights = np.ones_like(pred)\n    corr = weighted_correlation(label, pred, dummy_weights)\n    return corr\n\ndef neutralize_series(series : pd.Series, by : pd.Series, proportion=1.0):\n    \"\"\"\n    neutralize pandas series (originally from the Numerai Tournament)\n    \"\"\"\n    scores = np.nan_to_num(series.values).reshape(-1, 1)\n    exposures = np.nan_to_num(by.values).reshape(-1, 1)\n    exposures = np.hstack((exposures, np.array([np.mean(np.nan_to_num(series.values))] * len(exposures)).reshape(-1, 1)))\n    correction = proportion * (exposures.dot(np.linalg.lstsq(exposures, scores)[0]))\n    corrected_scores = scores - correction\n    neutralized = pd.Series(corrected_scores.ravel(), index=series.index)\n    return neutralized\n\ndef feature_exposures(df, prediction_name = 'Target'):\n    feature_names = features\n    exposures = []\n    for f in feature_names:\n        fe = np.corrcoef(np.nan_to_num(df[prediction_name].values), np.nan_to_num(df[f].values))[0, 1]\n        exposures.append(fe)\n    return np.array(exposures)\n\ndef max_feature_exposure(df): return np.max(np.abs(feature_exposures(df)))\ndef feature_exposure(df): return np.sqrt(np.mean(np.square(feature_exposures(df))))","metadata":{"_uuid":"bad4a334-9061-4729-a53b-efb92ab739b6","_cell_guid":"e3dca178-1764-4dfb-84f4-6af8ac785fdc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-04T15:27:58.164474Z","iopub.execute_input":"2022-01-04T15:27:58.164693Z","iopub.status.idle":"2022-01-04T15:28:00.852045Z","shell.execute_reply.started":"2022-01-04T15:27:58.164666Z","shell.execute_reply":"2022-01-04T15:28:00.851352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df, row=False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    \n    \n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n\n    return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=10)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:22.169321Z","iopub.execute_input":"2022-01-04T15:16:22.169567Z","iopub.status.idle":"2022-01-04T15:16:22.185882Z","shell.execute_reply.started":"2022-01-04T15:16:22.169538Z","shell.execute_reply":"2022-01-04T15:16:22.185076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_feather(\"../input/filledtraindata/train.feather\")\ndf['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\nasset_df = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\", index_col=\"Asset_Name\")","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:22.187766Z","iopub.execute_input":"2022-01-04T15:16:22.188041Z","iopub.status.idle":"2022-01-04T15:16:36.506884Z","shell.execute_reply.started":"2022-01-04T15:16:22.187997Z","shell.execute_reply":"2022-01-04T15:16:36.506173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import TimeSeriesSplit\ncv = TimeSeriesSplit(n_splits=N_SPLIT, test_size=TEST_DAY * 24 * 60, gap=GAP_DAY * 24 * 60, max_train_size=None if TRAIN_DAY < 0 else TRAIN_DAY * 24 * 60)","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:36.507931Z","iopub.execute_input":"2022-01-04T15:16:36.508704Z","iopub.status.idle":"2022-01-04T15:16:36.513930Z","shell.execute_reply.started":"2022-01-04T15:16:36.508664Z","shell.execute_reply":"2022-01-04T15:16:36.513093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score_for_one_symbol(all_df, asset_id, dry_run=False, model_params={}, dump_root=\"ckpt\"):\n    symbol_df = df[all_df.Asset_ID == asset_id].fillna(method=\"ffill\").dropna()\n    train_score_by_cv = [0] * N_SPLIT\n    test_score_by_cv = [0] * N_SPLIT\n    train_size_by_cv = [0] * N_SPLIT\n    test_size_by_cv = [0] * N_SPLIT\n    iter_by_cv = [0] * N_SPLIT\n    for i, (train_index, test_index) in enumerate(cv.split(symbol_df)):\n        train_size = len(train_index)\n        test_size = len(test_index)\n        start_index, end_index = train_index[0], test_index[-1]  # make the dataframe continuous in time\n        df_proc = get_features(symbol_df.iloc[start_index: end_index])\n        train_features, train_target = df_proc.iloc[:train_size], symbol_df[\"Target\"].iloc[train_index]\n        test_features, test_target = df_proc.iloc[-test_size:], symbol_df[\"Target\"].iloc[test_index]\n        train_features = train_features.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n        test_features = test_features.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n        train_set = lgb.Dataset(train_features, label=train_target)\n        test_set = lgb.Dataset(test_features, label=test_target)\n        # continuous\n        assert symbol_df.iloc[start_index: end_index].shape[0] == len(range(symbol_df[\"timestamp\"].iloc[start_index], symbol_df[\"timestamp\"].iloc[end_index], 60))\n        assert len(train_features) == len(train_target)\n        assert len(test_features) == len(test_target)\n        booster = lgb.train(train_set=train_set, params=model_params, valid_sets=[test_set], feval=pearson_eval,)\n        corr_train = validate_one_symble(booster, train_features, train_target)\n        corr_test = validate_one_symble(booster, test_features, test_target)\n#         print(\"Score on Train[{}]: {:.4f}\".format(i, corr_train))\n#         print(\"Score on Test[{}]: {:.4f}\".format(i, corr_test))\n        train_score_by_cv[i] = float(corr_train)\n        test_score_by_cv[i] = float(corr_test)\n        train_size_by_cv[i] = int(train_size)\n        test_size_by_cv[i] = int(test_size)\n        iter_by_cv[i] = booster.best_iteration\n        str_path = os.path.join(os.getcwd(), dump_root, asset_id, str(i))\n        os.makedirs(str_path, exist_ok=True)\n        model_str = booster.model_to_string()\n        with open(os.path.join(str_path, \"lgb.ckpt\"), \"w\") as f:\n            f.write(model_str)\n        \n        if dry_run:\n            break\n    avg_train_score = sum(train_score_by_cv) / N_SPLIT\n    avg_test_score = sum(test_score_by_cv) / N_SPLIT\n    best_iteration = booster.best_iteration\n    meta = {\n            \"train_score\": train_score_by_cv,\n            \"test_score\": test_score_by_cv,\n            \"train_size_by_cv\": train_size_by_cv,\n            \"test_size_by_cv\": test_size_by_cv,\n            \"model_params\": model_params,\n            \"avg_train_score\": avg_train_score,\n            \"avg_test_score\": avg_test_score,\n            \"iter_by_cv\": iter_by_cv\n        }\n        \n    meta_path = os.path.join(os.getcwd(), dump_root, asset_id, \"lgb_meta.json\")\n    with open(meta_path, \"w\") as f:\n        f.write(json.dumps(meta, indent=2))\n    return avg_train_score, avg_test_score, meta","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:36.515744Z","iopub.execute_input":"2022-01-04T15:16:36.516086Z","iopub.status.idle":"2022-01-04T15:16:36.539165Z","shell.execute_reply.started":"2022-01-04T15:16:36.516039Z","shell.execute_reply":"2022-01-04T15:16:36.538258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = asset_df[\"Weight\"]\nweights = weights / weights.sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:36.540712Z","iopub.execute_input":"2022-01-04T15:16:36.541046Z","iopub.status.idle":"2022-01-04T15:16:36.555552Z","shell.execute_reply.started":"2022-01-04T15:16:36.540995Z","shell.execute_reply":"2022-01-04T15:16:36.554639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_score_by_symbol = {}\ntest_score_by_symbol = {}\n\nfor asset_id in df.Asset_ID.unique():\n    if asset_id in SKIPS:\n        print(\"Skip \", asset_id)\n        continue\n    print(asset_id + \"\\n***\")\n    train_score, test_score, meta = get_score_for_one_symbol(df, asset_id, dry_run=False, model_params=MODEL_PARAMS, dump_root=CKPT)\n    train_score_by_symbol[asset_id] = train_score\n    test_score_by_symbol[asset_id] = test_score\n    \n    print(meta)\n    print(\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:36.557053Z","iopub.execute_input":"2022-01-04T15:16:36.557467Z","iopub.status.idle":"2022-01-04T15:17:04.627241Z","shell.execute_reply.started":"2022-01-04T15:16:36.557427Z","shell.execute_reply":"2022-01-04T15:17:04.625775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train_score = sum([score * weights[s] for s, score in train_score_by_symbol.items()])\nfinal_test_score = sum([score * weights[s] for s, score in test_score_by_symbol.items()])\nprint(\"avg. model score on train: {:.4f}\".format(final_train_score))\nprint(\"avg. model score on test: {:.4f}\".format(final_test_score))","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:17:04.628406Z","iopub.status.idle":"2022-01-04T15:17:04.629067Z","shell.execute_reply.started":"2022-01-04T15:17:04.628842Z","shell.execute_reply":"2022-01-04T15:17:04.628870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_by_symbol = pd.DataFrame({\"train_score\": train_score_by_symbol, \"test_score\": test_score_by_symbol}).sort_values(by=\"train_score\")","metadata":{"execution":{"iopub.status.busy":"2022-01-03T08:15:21.255871Z","iopub.execute_input":"2022-01-03T08:15:21.25622Z","iopub.status.idle":"2022-01-03T08:15:21.264873Z","shell.execute_reply.started":"2022-01-03T08:15:21.256173Z","shell.execute_reply":"2022-01-03T08:15:21.263961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_by_symbol","metadata":{"execution":{"iopub.status.busy":"2022-01-03T08:15:22.89688Z","iopub.execute_input":"2022-01-03T08:15:22.897782Z","iopub.status.idle":"2022-01-03T08:15:22.918013Z","shell.execute_reply.started":"2022-01-03T08:15:22.89774Z","shell.execute_reply":"2022-01-03T08:15:22.91647Z"},"trusted":true},"execution_count":null,"outputs":[]}]}