{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==0.24.2","metadata":{"_uuid":"35d9ff38-86eb-4ce6-94e1-0fef4fc74b31","_cell_guid":"97deee1c-1a28-4ff0-abde-197857d911ca","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T21:39:18.886084Z","iopub.execute_input":"2022-01-17T21:39:18.886442Z","iopub.status.idle":"2022-01-17T21:39:35.562629Z","shell.execute_reply.started":"2022-01-17T21:39:18.886345Z","shell.execute_reply":"2022-01-17T21:39:35.561834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CrossValidation Method\n- In TimeSeries, TimeSeriesSplit usually is used for cross-validation. However, this will limit the number of training data used for training.\n- According to \"Mörke, Mathis. \"Marcos López de Prado: Advances in financial machine learning.\" (2019): 491-493.\", we can also use PurgedKFold to do the cross-validation. In this schema, the purpose is to find the model performance under different market temperature. So we need to mannually find a split of market and test the model performance. Also, in order to prevent look-ahead bias, we use one month gap before and after the test data size.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"TEST_DAY = 3 * 30\n# train_day = 6 * 30\nTRAIN_DAY = -1\nGAP_DAY = 15\nN_SPLIT = 5\nCKPT = \"ckpt\"\n# SKIPS = ['Maker', \"Monero\", \"Stellar\"]\nSKIPS = []\n\nMODEL_PARAMS = {\n    \"n_estimators\": 1000,\n    \"early_stopping_round\": 50,\n    \"max_depth\": 4,  # choose a very shallow depth to ovoid overfitting.\n    \"random_seed\": 2021,\n    \"learning_rate\": 1e-3,\n    \"colsample_bytree\": 0.3,  # For the most of the time, trader only looks at <= 5 features to make decision. Accordingly, we limite the feature-wise sample size.\n    \"subsample\": 0.8,\n    \"metric\": \"custom\",\n    \"verbosity\": -1,\n    \"min_data_in_leaf\": 100,\n    \"device\": \"gpu\"\n}","metadata":{"_uuid":"ec27d8e6-e75b-4f1e-84da-301455db157f","_cell_guid":"bbd9a56d-875f-4c44-a011-a1d4df4f7c48","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T21:39:35.564263Z","iopub.execute_input":"2022-01-17T21:39:35.564599Z","iopub.status.idle":"2022-01-17T21:39:35.571614Z","shell.execute_reply.started":"2022-01-17T21:39:35.564557Z","shell.execute_reply":"2022-01-17T21:39:35.57077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb \nimport sklearn\nimport os\nimport json\nfrom scipy.stats import pearsonr\nimport logging\n\ndef pearson_eval(preds, train_data):\n    \"\"\"customized lgb evaluation method \"\"\"\n    labels = np.nan_to_num(train_data.get_label())\n    return 'corr', pearsonr(labels, np.nan_to_num(preds))[0], True\n\nlogger = logging.getLogger()\nlogger.setLevel(logging.INFO)\nlgb.register_logger(logger)\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / (np.sqrt(var_a * var_b) + 1e-12)\n    return corr\n\ndef validate_one_symble(model, features, label):\n    pred = model.predict(features)\n    dummy_weights = np.ones_like(pred)\n    corr = weighted_correlation(label, pred, dummy_weights)\n    return corr\n\ndef neutralize_series(series : pd.Series, by : pd.Series, proportion=1.0):\n    \"\"\"\n    neutralize pandas series (originally from the Numerai Tournament)\n    \"\"\"\n    scores = np.nan_to_num(series.values).reshape(-1, 1)\n    exposures = np.nan_to_num(by.values).reshape(-1, 1)\n    exposures = np.hstack((exposures, np.array([np.mean(np.nan_to_num(series.values))] * len(exposures)).reshape(-1, 1)))\n    correction = proportion * (exposures.dot(np.linalg.lstsq(exposures, scores)[0]))\n    corrected_scores = scores - correction\n    neutralized = pd.Series(corrected_scores.ravel(), index=series.index)\n    return neutralized\n\ndef feature_exposures(df, prediction_name = 'Target'):\n    feature_names = features\n    exposures = []\n    for f in feature_names:\n        fe = np.corrcoef(np.nan_to_num(df[prediction_name].values), np.nan_to_num(df[f].values))[0, 1]\n        exposures.append(fe)\n    return np.array(exposures)\n\ndef max_feature_exposure(df): return np.max(np.abs(feature_exposures(df)))\ndef feature_exposure(df): return np.sqrt(np.mean(np.square(feature_exposures(df))))","metadata":{"_uuid":"bad4a334-9061-4729-a53b-efb92ab739b6","_cell_guid":"e3dca178-1764-4dfb-84f4-6af8ac785fdc","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-17T21:39:35.572819Z","iopub.execute_input":"2022-01-17T21:39:35.573044Z","iopub.status.idle":"2022-01-17T21:39:37.518288Z","shell.execute_reply.started":"2022-01-17T21:39:35.573015Z","shell.execute_reply":"2022-01-17T21:39:37.517364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\n# A utility function to build features from the original df\n# It works for rows to, so we can reutilize it.\ndef get_features(df, row=False):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    \n    \n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    if row:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean()\n    else:\n        df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    \n    df_feat['High/Mean'] = df_feat['High'] / df_feat['Mean']\n    df_feat['Low/Mean'] = df_feat['Low'] / df_feat['Mean']\n    df_feat['Volume/Count'] = df_feat['Volume'] / (df_feat['Count'] + 1)\n\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dayofweek \n        df_feat[\"day\"] = times.day \n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n        df_feat[\"dayofweek\"] = times.dt.dayofweek \n        df_feat[\"day\"] = times.dt.day \n    #df_feat.drop(columns=[\"time\"],errors=\"ignore\",inplace=True)  # keep original epoch time, drop string\n\n    return df_feat\n\n\ndef get_Xy_and_model_for_asset(df_train, asset_id):\n    df = df_train[df_train[\"Asset_ID\"] == asset_id]\n    \n    # TODO: Try different features here!\n    df_proc = get_features(df)\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n    \n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n\n    # TODO: Try different models here!\n    model = LGBMRegressor(n_estimators=10)\n    model.fit(X, y)\n    return X, y, model","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:39:37.520903Z","iopub.execute_input":"2022-01-17T21:39:37.521226Z","iopub.status.idle":"2022-01-17T21:39:37.537817Z","shell.execute_reply.started":"2022-01-17T21:39:37.521185Z","shell.execute_reply":"2022-01-17T21:39:37.536854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_feather(\"../input/filledtraindata/train.feather\")\ndf['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\nasset_df = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\", index_col=\"Asset_Name\")","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:39:37.539265Z","iopub.execute_input":"2022-01-17T21:39:37.539549Z","iopub.status.idle":"2022-01-17T21:39:56.689282Z","shell.execute_reply.started":"2022-01-17T21:39:37.53952Z","shell.execute_reply":"2022-01-17T21:39:56.688477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = asset_df[\"Weight\"]\nweights = weights / weights.sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:39:56.690528Z","iopub.execute_input":"2022-01-17T21:39:56.69075Z","iopub.status.idle":"2022-01-17T21:39:56.695948Z","shell.execute_reply.started":"2022-01-17T21:39:56.690721Z","shell.execute_reply":"2022-01-17T21:39:56.69511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"close_df = df.pivot(index=\"datetime\", columns=\"Asset_ID\", values=\"Close\")\nclose_df = close_df.fillna(method=\"ffill\")\nclose_df = close_df.pct_change(1)\nweighted_ret = close_df.multiply(other=weights).sum(axis=1).cumsum()\nweighted_ret.plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:39:56.6973Z","iopub.execute_input":"2022-01-17T21:39:56.69756Z","iopub.status.idle":"2022-01-17T21:40:27.159159Z","shell.execute_reply.started":"2022-01-17T21:39:56.697531Z","shell.execute_reply":"2022-01-17T21:40:27.158367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period1 = slice(\"2018-1-1\", \"2018-11-1\")  # 178560\nweighted_ret.loc[period1].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:27.160503Z","iopub.execute_input":"2022-01-17T21:40:27.160768Z","iopub.status.idle":"2022-01-17T21:40:30.221432Z","shell.execute_reply.started":"2022-01-17T21:40:27.16074Z","shell.execute_reply":"2022-01-17T21:40:30.220486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period2 = slice(\"2019-1-1\", \"2019-7-1\")\nweighted_ret.loc[period2].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:30.222655Z","iopub.execute_input":"2022-01-17T21:40:30.222944Z","iopub.status.idle":"2022-01-17T21:40:31.82092Z","shell.execute_reply.started":"2022-01-17T21:40:30.222912Z","shell.execute_reply":"2022-01-17T21:40:31.820049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period3 = slice(\"2019-8-1\", \"2019-12-1\")\nweighted_ret.loc[period3].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:31.82351Z","iopub.execute_input":"2022-01-17T21:40:31.823835Z","iopub.status.idle":"2022-01-17T21:40:33.483228Z","shell.execute_reply.started":"2022-01-17T21:40:31.823802Z","shell.execute_reply":"2022-01-17T21:40:33.482338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period4 = slice(\"2020-1-1\", \"2021-5-1\")\nweighted_ret.loc[period4].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:33.484236Z","iopub.execute_input":"2022-01-17T21:40:33.484456Z","iopub.status.idle":"2022-01-17T21:40:37.768011Z","shell.execute_reply.started":"2022-01-17T21:40:33.484429Z","shell.execute_reply":"2022-01-17T21:40:37.767068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period5 = slice(\"2021-6-1\", \"2021-10-1\")\nweighted_ret.loc[period5].plot()","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:37.769378Z","iopub.execute_input":"2022-01-17T21:40:37.769683Z","iopub.status.idle":"2022-01-17T21:40:38.814918Z","shell.execute_reply.started":"2022-01-17T21:40:37.76964Z","shell.execute_reply":"2022-01-17T21:40:38.813972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_SCORE_DF = pd.DataFrame(index=asset_df.index, columns=[period1.start, period2.start, period3.start, period4.start, period5.start])\nTRAIN_SCORE_DF = pd.DataFrame(index=asset_df.index, columns=[period1.start, period2.start, period3.start, period4.start, period5.start])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score_for_one_symbol_new_cv(all_df, asset_id, dry_run=False, model_params={}, dump_root=\"ckpt\"):\n    symbol_df = all_df[all_df.Asset_ID == asset_id].fillna(method=\"ffill\").dropna()\n    symbol_df = symbol_df.set_index(\"datetime\")\n    if asset_id == \"Maker\":\n        symbol_df = symbol_df.loc[\"2020-08-04\":]\n    elif asset_id == \"Monero\":\n        symbol_df = symbol_df.loc[\"2018-11-05\":]\n    elif asset_id == \"Stellar\":\n        symbol_df = symbol_df.loc[\"2018-07-14\":]\n    train_score_by_cv = [0] * N_SPLIT\n    test_score_by_cv = [0] * N_SPLIT\n    train_size_by_cv = [0] * N_SPLIT\n    test_size_by_cv = [0] * N_SPLIT\n    test_period_by_cv = [0] * N_SPLIT\n    test_type_by_cv = [0] * N_SPLIT\n    iter_by_cv = [0] * N_SPLIT\n    df_proc = get_features(symbol_df)\n    bulls = [period4, period2]\n    bears = [period3]\n    neutral = [period1, period5]\n    for i, period in enumerate([period1, period2, period3, period4, period5]):\n        print(period)\n        train_features, train_target = df_proc.loc[period], symbol_df[\"Target\"].loc[period]\n        test_features, test_target = df_proc.loc[period], symbol_df[\"Target\"].loc[period]\n        if test_features.size == 0:\n            continue\n        part1 = pd.Timestamp(period.start) - pd.Timedelta(\"1m\")\n        part2 = pd.Timestamp(period.stop) + pd.Timedelta(\"1m\")\n        dfs = []\n        targets = []\n        _df1 = df_proc.loc[:part1]\n        _df2 = df_proc.loc[part2:]\n        if _df1.size > 0:\n            dfs.append(_df1)\n            targets.append(symbol_df[\"Target\"].loc[:part1])\n        if _df2.size > 0:\n            dfs.append(_df2)\n            targets.append(symbol_df[\"Target\"].loc[part2:])\n        if len(dfs) == 2:\n            train_features = pd.concat(dfs)\n            train_target = pd.concat(targets)\n        elif len(dfs) == 1:\n            train_features = dfs[0]\n            train_target = targets[0]\n        else:\n            continue\n        train_size = len(train_features)\n        test_size = len(test_features)\n        train_features = train_features.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n        test_features = test_features.replace([np.inf, -np.inf], np.nan).dropna(how=\"any\")\n        train_set = lgb.Dataset(train_features, label=train_target)\n        test_set = lgb.Dataset(test_features, label=test_target)\n        # continuous\n        assert len(train_features) == len(train_target)\n        assert len(test_features) == len(test_target)\n        booster = lgb.train(train_set=train_set, params=model_params, valid_sets=[test_set], feval=pearson_eval)\n        corr_train = validate_one_symble(booster, train_features, train_target)\n        corr_test = validate_one_symble(booster, test_features, test_target)\n#         print(\"Score on Train[{}]: {:.4f}\".format(i, corr_train))\n#         print(\"Score on Test[{}]: {:.4f}\".format(i, corr_test))\n        TEST_SCORE_DF.loc[asset_id, period.start] = float(corr_test)\n        TRAIN_SCORE_DF.loc[asset_id, period.start] = float(corr_train)\n        train_score_by_cv[i] = float(corr_train)\n        test_score_by_cv[i] = float(corr_test)\n        train_size_by_cv[i] = int(train_size)\n        test_size_by_cv[i] = int(test_size)\n        test_period_by_cv[i] = [period.start, period.stop]\n        if period in bulls:\n            test_type_by_cv[i] = \"bull\"\n        elif period in bears:\n            test_type_by_cv[i] = \"bear\"\n        else:\n            test_type_by_cv[i] = \"neutral\"\n        iter_by_cv[i] = booster.best_iteration\n        str_path = os.path.join(os.getcwd(), dump_root, asset_id, str(i))\n        os.makedirs(str_path, exist_ok=True)\n        model_str = booster.model_to_string()\n        with open(os.path.join(str_path, \"lgb.ckpt\"), \"w\") as f:\n            f.write(model_str)\n        \n        if dry_run:\n            break\n    avg_train_score = sum(train_score_by_cv) / N_SPLIT\n    avg_test_score = sum(test_score_by_cv) / N_SPLIT\n    best_iteration = booster.best_iteration\n    meta = {\n            \"train_score\": train_score_by_cv,\n            \"test_score\": test_score_by_cv,\n            \"train_size_by_cv\": train_size_by_cv,\n            \"test_size_by_cv\": test_size_by_cv,\n            \"test_type_by_cv\": test_type_by_cv,\n            \"test_period_by_cv\": test_period_by_cv,\n            \"model_params\": model_params,\n            \"avg_train_score\": avg_train_score,\n            \"avg_test_score\": avg_test_score,\n            \"iter_by_cv\": iter_by_cv\n        }\n        \n    meta_path = os.path.join(os.getcwd(), dump_root, asset_id, \"lgb_meta.json\")\n    with open(meta_path, \"w\") as f:\n        f.write(json.dumps(meta, indent=2))\n    return avg_train_score, avg_test_score, meta","metadata":{"execution":{"iopub.status.busy":"2022-01-17T21:40:38.816274Z","iopub.execute_input":"2022-01-17T21:40:38.81654Z","iopub.status.idle":"2022-01-17T21:40:38.844915Z","shell.execute_reply.started":"2022-01-17T21:40:38.816509Z","shell.execute_reply":"2022-01-17T21:40:38.844066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_score_by_symbol = {}\ntest_score_by_symbol = {}\n\nfor asset_id in df.Asset_ID.unique():\n#     if asset_id in SKIPS:\n#         print(\"Skip \", asset_id)\n#         continue\n    print(asset_id + \"\\n***\")\n#     train_score, test_score, meta = get_score_for_one_symbol(df, asset_id, dry_run=False, model_params=MODEL_PARAMS, dump_root=CKPT)\n    train_score, test_score, meta = get_score_for_one_symbol_new_cv(df, asset_id, dry_run=False, model_params=MODEL_PARAMS, dump_root=CKPT)\n    train_score_by_symbol[asset_id] = train_score\n    test_score_by_symbol[asset_id] = test_score\n    \n    print(meta)\n    print(\"\\n\")\n    ","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:16:36.557053Z","iopub.execute_input":"2022-01-04T15:16:36.557467Z","iopub.status.idle":"2022-01-04T15:17:04.627241Z","shell.execute_reply.started":"2022-01-04T15:16:36.557427Z","shell.execute_reply":"2022-01-04T15:17:04.625775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SCORE_DF","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_SCORE_DF","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SCORE_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"train_score_df.csv\"))\nTEST_SCORE_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"test_score_df.csv\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train_score = sum([score * weights[s] for s, score in train_score_by_symbol.items()])\nfinal_test_score = sum([score * weights[s] for s, score in test_score_by_symbol.items()])\nprint(\"avg. model score on train: {:.4f}\".format(final_train_score))\nprint(\"avg. model score on test: {:.4f}\".format(final_test_score))","metadata":{"execution":{"iopub.status.busy":"2022-01-04T15:17:04.628406Z","iopub.status.idle":"2022-01-04T15:17:04.629067Z","shell.execute_reply.started":"2022-01-04T15:17:04.628842Z","shell.execute_reply":"2022-01-04T15:17:04.62887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_by_symbol = pd.DataFrame({\"train_score\": train_score_by_symbol, \"test_score\": test_score_by_symbol}).sort_values(by=\"train_score\")","metadata":{"execution":{"iopub.status.busy":"2022-01-03T08:15:21.255871Z","iopub.execute_input":"2022-01-03T08:15:21.25622Z","iopub.status.idle":"2022-01-03T08:15:21.264873Z","shell.execute_reply.started":"2022-01-03T08:15:21.256173Z","shell.execute_reply":"2022-01-03T08:15:21.263961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_by_symbol","metadata":{"execution":{"iopub.status.busy":"2022-01-03T08:15:22.89688Z","iopub.execute_input":"2022-01-03T08:15:22.897782Z","iopub.status.idle":"2022-01-03T08:15:22.918013Z","shell.execute_reply.started":"2022-01-03T08:15:22.89774Z","shell.execute_reply":"2022-01-03T08:15:22.91647Z"},"trusted":true},"execution_count":null,"outputs":[]}]}