{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install scikit-learn==0.24.2\n!pip install talib-binary","metadata":{"_uuid":"1f1ac090-94b4-452b-bde6-e5fa63d01453","_cell_guid":"045ba1b0-fa45-4ce2-89e6-8b34a4308d35","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-31T01:41:23.664609Z","iopub.execute_input":"2022-01-31T01:41:23.665085Z","iopub.status.idle":"2022-01-31T01:41:52.831527Z","shell.execute_reply.started":"2022-01-31T01:41:23.664995Z","shell.execute_reply":"2022-01-31T01:41:52.830263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CrossValidation Method\n- In TimeSeries, TimeSeriesSplit usually is used for cross-validation. However, this will limit the number of training data used for training.\n- According to \"Mörke, Mathis. \"Marcos López de Prado: Advances in financial machine learning.\" (2019): 491-493.\", we can also use PurgedKFold to do the cross-validation. In this schema, the purpose is to find the model performance under different market temperature. So we need to mannually find a split of market and test the model performance. Also, in order to prevent look-ahead bias, we use one month gap before and after the test data size.\n\n## Feature Engineering\n- The engineering techniques are described here https://www.kaggle.com/axzhang/pipeline-building-featureengineering-stage1","metadata":{"_uuid":"a36391dd-d38b-48ce-897a-6165e2be2ab2","_cell_guid":"78992eaa-0880-4c6c-8af7-aad27046e7f0","trusted":true}},{"cell_type":"code","source":"TEST_DAY = 3 * 30\n# train_day = 6 * 30\nTRAIN_DAY = -1\nGAP_DAY = 15\nN_SPLIT = 6\nCKPT = \"ckpt\"\nSKIPS = []\nEPS = 1e-18\n\nMODEL_PARAMS = {\n    \"n_estimators\": 1000,\n    \"early_stopping_round\": 50,\n    \"max_depth\": 4,  # choose a very shallow depth to ovoid overfitting.\n    \"random_seed\": 2021,\n    \"learning_rate\": 1e-3,\n    \"colsample_bytree\": 0.3,  # For the most of the time, trader only looks at <= 5 features to make decision. Accordingly, we limite the feature-wise sample size.\n    \"subsample\": 0.3,\n    \"metric\": \"custom\",\n    \"verbosity\": -1,\n    \"min_data_in_leaf\": 100,\n    \"device\": \"gpu\"\n}","metadata":{"_uuid":"5924dbf2-81fd-4a6d-bef2-b62334d413c9","_cell_guid":"36bd542b-c1e1-48ee-acdc-b3ce8c96782e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-31T01:41:52.834932Z","iopub.execute_input":"2022-01-31T01:41:52.836865Z","iopub.status.idle":"2022-01-31T01:41:52.844872Z","shell.execute_reply.started":"2022-01-31T01:41:52.836810Z","shell.execute_reply":"2022-01-31T01:41:52.843448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb \nimport sklearn\nimport re\nimport os\nimport json\nimport talib\nimport gc\nimport pickle\nfrom tqdm.notebook import tqdm\nfrom scipy.stats import pearsonr\nimport logging\npd.set_option('display.max_rows', 200)\ndef pearson_eval(preds, train_data):\n    \"\"\"customized lgb evaluation method \"\"\"\n    labels = np.nan_to_num(train_data.get_label())\n    return 'corr', pearsonr(labels, np.nan_to_num(preds))[0], True\n\n# logger = logging.getLogger()\n# logger.setLevel(logging.INFO)\n# lgb.register_logger(logger)\ndef weighted_correlation(a, b, weights):\n    w = np.ravel(weights)\n    a = np.ravel(a)\n    b = np.ravel(b)\n\n    sum_w = np.sum(w)\n    mean_a = np.sum(a * w) / sum_w\n    mean_b = np.sum(b * w) / sum_w\n    var_a = np.sum(w * np.square(a - mean_a)) / sum_w\n    var_b = np.sum(w * np.square(b - mean_b)) / sum_w\n\n    cov = np.sum((a * b * w)) / np.sum(w) - mean_a * mean_b\n    corr = cov / (np.sqrt(var_a * var_b) + 1e-12)\n    return corr\n\ndef validate_one_symble(model, features, label):\n    pred = model.predict(features)\n    dummy_weights = np.ones_like(pred)\n    corr = weighted_correlation(label, pred, dummy_weights)\n    return corr\n\ndef neutralize_series(series : pd.Series, by : pd.Series, proportion=1.0):\n    \"\"\"\n    neutralize pandas series (originally from the Numerai Tournament)\n    \"\"\"\n    scores = np.nan_to_num(series.values).reshape(-1, 1)\n    exposures = np.nan_to_num(by.values).reshape(-1, 1)\n    exposures = np.hstack((exposures, np.array([np.mean(np.nan_to_num(series.values))] * len(exposures)).reshape(-1, 1)))\n    correction = proportion * (exposures.dot(np.linalg.lstsq(exposures, scores)[0]))\n    corrected_scores = scores - correction\n    neutralized = pd.Series(corrected_scores.ravel(), index=series.index)\n    return neutralized\n\ndef feature_exposures(df, prediction_name = 'Target'):\n    feature_names = features\n    exposures = []\n    for f in feature_names:\n        fe = np.corrcoef(np.nan_to_num(df[prediction_name].values), np.nan_to_num(df[f].values))[0, 1]\n        exposures.append(fe)\n    return np.array(exposures)\n\ndef max_feature_exposure(df): return np.max(np.abs(feature_exposures(df)))\ndef feature_exposure(df): return np.sqrt(np.mean(np.square(feature_exposures(df))))","metadata":{"_uuid":"99b6e14c-4b48-4351-8e5c-006987bba1f4","_cell_guid":"2d29f90e-9d1a-42fc-8940-5fe4cf4b7f3b","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-01-31T01:41:52.846921Z","iopub.execute_input":"2022-01-31T01:41:52.847643Z","iopub.status.idle":"2022-01-31T01:41:56.557971Z","shell.execute_reply.started":"2022-01-31T01:41:52.847595Z","shell.execute_reply":"2022-01-31T01:41:56.557014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_feature_name(func_name, meta):\n    if \"timeperiod\" in meta:\n        if isinstance(meta[\"timeperiod\"], str):\n            name = func_name + \"_\" + \"_\".join([str(i) for i in eval(meta[\"timeperiod\"])])\n        else:\n            name = func_name + \"_\" + str(meta[\"timeperiod\"])\n    else:\n        name = func_name\n    return name\n\ndef read_feature_config(symbol):\n    with open(os.path.join(\"../input/bestparameter\", \"best_period_{}.json\".format(symbol))) as f:\n        config = json.load(f)\n    new_config = {}\n    for func_name, setting in config.items():\n        setting[\"func_name\"] = func_name\n        feature_name = get_feature_name(func_name, setting)\n        new_config[feature_name] = setting\n    return new_config","metadata":{"_uuid":"ff01e030-cd94-4058-981a-30aeb2321f48","_cell_guid":"413812e1-a3f9-4b6e-8a19-33827c36120e","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.559458Z","iopub.execute_input":"2022-01-31T01:41:56.559788Z","iopub.status.idle":"2022-01-31T01:41:56.569157Z","shell.execute_reply.started":"2022-01-31T01:41:56.559734Z","shell.execute_reply":"2022-01-31T01:41:56.567616Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def RET(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_C(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_H(df, n):\n    return df['High'].pct_change(n)\n    \ndef RET_L(df, n):\n    return df['Low'].pct_change(n)\n    \ndef RET_O(df, n):\n    return df['Open'].pct_change(n)\n    \ndef RET_V(df, n):\n    return df['Volume'].pct_change(n)\n    \ndef RET_VWAP(df, n):\n    return df['VWAP'].pct_change(n)\n    \ndef RET_Cnt(df, n):\n    return df['Count'].pct_change(n)\n\ndef STD(df, n):\n    return df['Close'].pct_change(1).rolling(n).std()\n\ndef RET_STD(df, n):\n    return RET(df, n) * STD(df, n)\n\ndef RSI(df, n):\n    return talib.RSI(df['Close'], n)\n\ndef ATR(df, n):\n    return talib.ATR(df[\"High\"], df.Low, df.Close, n)\n\ndef MFI(df, n):\n    return talib.MFI(df['High'], df['Low'], df['Close'], df['Volume'], n)\n\ndef VOL(df, n):\n    ret = df['Close'].pct_change(1)\n    return np.sqrt((ret ** 2).rolling(n).mean())\n\ndef TRIX(df, n):\n    return talib.TRIX(df['Close'], n)\n\ndef MACD(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[0]\n\ndef MACD_HIST(df, fast, slow):\n    return talib.MACD(df.Close, fast, slow)[2]\n\ndef DEMA(df, n1, n2):\n    return np.log(df['Close'].rolling(n1).mean() / (df['Close'].rolling(n2).mean() + EPS))\n\ndef EFFICIENCY(df, n):\n    speed = (df.Close - df.Close.shift(n))\n    volatility = (df.Close - df.Close.shift(1)).abs().rolling(n).sum()\n    return speed / (volatility+ EPS)\n\ndef EI(df, n):\n    neg = df.Close - df.Low.rolling(n).min()\n    high = df.High.rolling(n).max() - df.Close\n    ei = high / (neg + EPS)\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME(df):\n    return (df.Volume > df.Volume.shift(1)).astype(float)\n\ndef EVENT_MACROSS(df, fast, slow):\n    fast_s = df.Close.rolling(fast).mean()\n    slow_s = df.Close.rolling(slow).mean()\n    return (fast_s > slow_s).astype(float)\n\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']","metadata":{"_uuid":"77e46a55-9656-4b15-a4b4-7ce8300cf24e","_cell_guid":"60738a2e-63be-4f0e-9c5b-9392c5bb25c0","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.572455Z","iopub.execute_input":"2022-01-31T01:41:56.573100Z","iopub.status.idle":"2022-01-31T01:41:56.600319Z","shell.execute_reply.started":"2022-01-31T01:41:56.573055Z","shell.execute_reply":"2022-01-31T01:41:56.598921Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def RET_VEC(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_C_VEC(df, n):\n    return df['Close'].pct_change(n)\n\ndef RET_H_VEC(df, n):\n    return df['High'].pct_change(n)\n    \ndef RET_L_VEC(df, n):\n    return df['Low'].pct_change(n)\n    \ndef RET_O_VEC(df, n):\n    return df['Open'].pct_change(n)\n    \ndef RET_V_VEC(df, n):\n    return df['Volume'].pct_change(n)\n    \ndef RET_VWAP_VEC(df, n):\n    return df['VWAP'].pct_change(n)\n    \ndef RET_Cnt_VEC(df, n):\n    return df['Count'].pct_change(n)\n\ndef STD_VEC(df, n):\n    return df['Close'].pct_change(1).rolling(n).std()\n\ndef RET_STD_VEC(df, n):\n    return RET_VEC(df, n) * STD_VEC(df, n)\n\ndef RSI_VEC(df, n):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.RSI(close[c], n)\n    return feat\n\ndef ATR_VEC(df, n):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.ATR(df[\"High\"][c], df[\"Low\"][c], df[\"Close\"][c], n)\n    return feat\n\ndef MFI_VEC(df, n):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.MFI(df[\"High\"][c], df[\"Low\"][c], df[\"Close\"][c], df[\"Volume\"][c], n)\n    return feat\n\ndef VOL_VEC(df, n):\n    ret = df['Close'].pct_change(1)\n    return np.sqrt((ret ** 2).rolling(n).mean())\n\ndef TRIX_VEC(df, n):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.TRIX(close[c], n)\n    return feat\n\ndef MACD_VEC(df, fast, slow):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.MACD(close[c], fast, slow)[0]\n    return feat\n\ndef MACD_HIST_VEC(df, fast, slow):\n    close = df[\"Close\"]\n    feat = pd.DataFrame(index=close.index, columns=close.columns)\n    for c in close.columns:\n        feat[c] = talib.MACD(close[c], fast, slow)[2]\n    return feat\n\ndef DEMA_VEC(df, n1, n2):\n    return np.log(df['Close'].rolling(n1).mean() / (df['Close'].rolling(n2).mean() + EPS))\n\ndef EFFICIENCY_VEC(df, n):\n    speed = (df[\"Close\"] - df[\"Close\"].shift(n))\n    volatility = (df[\"Close\"] - df[\"Close\"].shift(1)).abs().rolling(n).sum()\n    return speed / (volatility+ EPS)\n\ndef EI_VEC(df, n):\n    neg = df[\"Close\"] - df[\"Low\"].rolling(n).min()\n    high = df[\"High\"].rolling(n).max() - df[\"Close\"]\n    ei = high / (neg + EPS)\n    ei = np.clip(ei, -100, 100)\n    return ei\n\ndef EVEMT_BIGGER_VOLUME_VEC(df):\n    return (df[\"Volume\"] > df[\"Volume\"].shift(1)).astype(float)\n\ndef EVENT_MACROSS_VEC(df, fast, slow):\n    fast_s = df[\"Close\"].rolling(fast).mean()\n    slow_s = df[\"Close\"].rolling(slow).mean()\n    return (fast_s > slow_s).astype(float)\n\ndef upper_shadow_VEC(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow_VEC(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']","metadata":{"_uuid":"176937d5-886b-44a2-8731-ebe1d078ea7a","_cell_guid":"b8b454d8-e9e0-44b5-a58b-602515315e1b","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.602280Z","iopub.execute_input":"2022-01-31T01:41:56.602719Z","iopub.status.idle":"2022-01-31T01:41:56.636770Z","shell.execute_reply.started":"2022-01-31T01:41:56.602672Z","shell.execute_reply":"2022-01-31T01:41:56.635392Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, row=False, config=None):\n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    d = {\"Count\": \"RET_Cnt\", \n         \"Open\": \"RET_O\", \n         \"Close\": \"RET_C\", \n         \"Volume\": \"RET_V\", \n         \"VWAP\": \"RET_VWAP\",\n         \"High\": \"RET_H\",\n         \"Cnt\": \"RET_Cnt\",\n         \"Low\": \"RET_L\",\n        }\n    if config is None:\n        config = {}\n    for col in ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']:\n        config[\"RET60_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 60}\n    \n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['target_lag'] = df[\"Target\"].shift(16)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    df_feat[\"MoM15\"] = df_feat[\"VWAP\"] - df_feat[\"VWAP\"].shift(15)\n    \n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"],unit=\"s\",infer_datetime_format=True)\n    if row:\n        df_feat[\"hour\"] = times.hour  # .dt\n    else:\n        df_feat[\"hour\"] = times.dt.hour  # .dt\n\n    # self-define feature engineering\n    if config is not None:\n        for feature_name, setting in config.items():\n            func_name = setting[\"func_name\"]\n            func = eval(func_name)\n            if \"timeperiod\" not in setting:\n                # no argument\n                feature: float = func(df_feat)\n            elif isinstance(setting.get(\"timeperiod\"), str):\n                args = eval(setting.get(\"timeperiod\"))\n                feature: float = func(df_feat, *args)\n            else:\n                feature: float = func(df_feat, setting.get(\"timeperiod\"))\n            df_feat[feature_name] = feature\n            \n    df_feat.pop(\"Close\")\n    df_feat.pop(\"High\")\n    df_feat.pop(\"Low\")\n    df_feat.pop(\"Open\")\n    df_feat.pop(\"VWAP\")\n    df_feat.pop(\"Volume\")\n    return df_feat","metadata":{"_uuid":"81df6198-7d5b-44e5-bf1a-0e028ed6ea35","_cell_guid":"f86b4011-e997-49bc-9cc5-0a24bebd83f4","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.639380Z","iopub.execute_input":"2022-01-31T01:41:56.640024Z","iopub.status.idle":"2022-01-31T01:41:56.657217Z","shell.execute_reply.started":"2022-01-31T01:41:56.639973Z","shell.execute_reply":"2022-01-31T01:41:56.656133Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features_vec(df, config=None):\n    removed_features = [\n    'Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', \"VOL_(.+?)\",\n    \"RET_6\",\"RET_5\",\"RET_24\",\"RET_9\",\"RET_90\",\"RET_92\",\"RET_81\",\"RET_96\",\"RET_40\",\n    \"STD_41\",\"STD_82\",\"STD_43\",\"STD_30\",\"STD_11\",\"STD_6\",\"STD_5\",\"STD_8\",\n    \"RET_STD_86\",\"RET_STD_69\",\"RET_STD_80\",\"RET_STD_10\",\"RET_STD_72\",\"RET_STD_45\",\"RET_STD_48\",\"RET_STD_26\",\"RET_STD_29\",\"RET_STD_25\",\"RET_STD_6\",\n    \"RSI_11\",\"RSI_32\",\"RSI_37\",\"RSI_38\",\"RSI_8\",\"RSI_42\",\"RSI_6\",\"RSI_5\",\"RSI_94\",\n    \"ATR_(.+?)\",\n    \"MFI_21\",\"MFI_13\",\"MFI_48\",\"MFI_38\",\"MFI_10\",\"MFI_69\",\"MFI_8\",\"MFI_6\",\"MFI_96\",\"MFI_99\",\"MFI_5\",\n    \"VOL_34\",\"VOL_82\",\"VOL_42\",\"VOL_96\",\"VOL_14\",\"VOL_10\",\"VOL_6\",\"VOL_5\",\"VOL_2\",\"VOL_4\",\n    \"TRIX_5\",\"TRIX_9\",\"TRIX_12\",\"TRIX_14\",\"TRIX_32\",\"TRIX_34\",\"TRIX_41\",\"TRIX_44\",\"TRIX_49\",\n    \"EFFICIENCY_15\",\"EFFICIENCY_14\",\"EFFICIENCY_13\",\"EFFICIENCY_72\",\"EFFICIENCY_81\",\"EFFICIENCY_92\",\"EFFICIENCY_96\",\"EFFICIENCY_97\",\n    \"EI_11\",\"EI_19\",\"EI_23\",\"EI_35\",\"EI_97\",\"EI_98\",\"EI_96\",\"EI_79\",\"EI_83\",\"EI_64\",\n    \"MACD_HIST_2_133\",\"MACD_HIST_2_183\",\"MACD_HIST_22_53\",\"MACD_HIST_2_53\",\"MACD_HIST_32_63\",\"MACD_HIST_22_183\",\"MACD_HIST_2_33\",\"MACD_HIST_32_183\",\"MACD_HIST_32_193\",\"MACD_HIST_42_163\",\"MACD_HIST_42_183\",\n    \"DEMA_2_28\",\"DEMA_2_63\",\"DEMA_2_108\",\"DEMA_2_138\",\"DEMA_2_143\",\"DEMA_2_153\",\"DEMA_8_13\",\"DEMA_16_158\",\"DEMA_42_98\",\"DEMA_32_98\",\"DEMA_48_133\",\"DEMA_48_143\",\"DEMA_44_63\",\n    \"MACD_4_8\",\"MACD_6_18\",\"MACD_12_13\",\"MACD_2_78\",\"MACD_2_83\",\"MACD_2_98\",\"MACD_2_113\",\"MACD_2_138\",\"MACD_2_153\",\"MACD_8_143\",\"MACD_48_118\",\n    \"EVENT_MACROSS_1_98\",\"EVENT_MACROSS_1_93\",\"EVENT_MACROSS_1_133\",\"EVENT_MACROSS_3_48\",\"EVENT_MACROSS_1_178\",\"EVENT_MACROSS_27_53\",\"EVENT_MACROSS_23_83\",\"EVENT_MACROSS_49_68\",\"EVENT_MACROSS_49_103\",\"EVENT_MACROSS_47_123\",\"EVENT_MACROSS_41_143\",\"EVENT_MACROSS_43_163\",\"EVENT_MACROSS_49_183\",\n    \"RET1_Count\",\"RET1_Open\",\"RET1_High\",\"RET1_Low\",\"RET1_Close\",\"RET1_Volume\",\"RET1_VWAP\"]\n    df_feat = {}\n    d = {\"Count\": \"RET_Cnt\", \n         \"Open\": \"RET_O\", \n         \"Close\": \"RET_C\", \n         \"Volume\": \"RET_V\", \n         \"VWAP\": \"RET_VWAP\",\n         \"High\": \"RET_H\",\n         \"Cnt\": \"RET_Cnt\",\n         \"Low\": \"RET_L\",\n        }\n    if config is None:\n        config = {}\n    for col in ['Count', 'Volume', 'VWAP']:\n        config[\"RET60_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 60}\n        config[\"RET240_{}\".format(col)] = {\"func_name\": d[col], \"timeperiod\": 240}\n    \n    df_feat['Upper_Shadow'] = upper_shadow_VEC(df)\n    df_feat['target_lag'] = df[\"Target\"].shift(16)\n    df_feat[\"MoM1\"] = df[\"Close\"] - df[\"Close\"].shift(1)\n    ## possible seasonality, datetime  features (unlikely to me meaningful, given very short time-frames)\n    ### to do: add cyclical features for seasonality\n    times = pd.to_datetime(df[\"timestamp\"].iloc[:, 0],unit=\"s\",infer_datetime_format=True)\n    df_feat[\"hour\"] = pd.DataFrame(np.broadcast_to(times.dt.hour.values.reshape(-1, 1), df[\"timestamp\"].shape), index=df[\"timestamp\"].index, columns=df[\"timestamp\"].columns)  # .dt\n\n    # self-define feature engineering\n    if config is not None:\n        for feature_name, setting in tqdm(config.items()):\n            remove = False\n            for feat in removed_features:\n                if re.match(feat, feature_name):\n                    remove = True\n                    break\n            if remove:\n                continue\n            args = ()\n            if \"timeperiod\" not in setting:\n                # no argument\n                pass\n            elif isinstance(setting.get(\"timeperiod\"), str):\n                args = eval(setting.get(\"timeperiod\"))\n            else:\n                args = (setting.get(\"timeperiod\"), )\n                \n            vec_func_name = setting[\"func_name\"] + \"_VEC\"\n            func = eval(vec_func_name)\n            df_feat[feature_name] = func(df, *args)\n\n    for feat in removed_features:\n        keys = list(df_feat.keys())\n        for k in keys:\n            if re.match(feat, k):\n                df_feat.pop(k)\n                print(\"remove \", k)\n    return df_feat","metadata":{"_uuid":"a1c1e02f-2fb5-4e05-9a94-7e42b4456705","_cell_guid":"7aae935d-7431-4e75-83df-a2eba442deaf","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.659112Z","iopub.execute_input":"2022-01-31T01:41:56.659409Z","iopub.status.idle":"2022-01-31T01:41:56.687074Z","shell.execute_reply.started":"2022-01-31T01:41:56.659373Z","shell.execute_reply":"2022-01-31T01:41:56.685983Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_feather(\"../input/filleddataset/train.feather\")\ndf['datetime'] = pd.to_datetime(df['timestamp'], unit='s')\ndf = df.set_index([\"Asset_ID\", \"datetime\"], drop=False)","metadata":{"_uuid":"dacbb785-49c7-4423-ae77-78af0e644f2c","_cell_guid":"403895e4-b279-47ff-8c0c-7e11ec48cc99","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:41:56.688866Z","iopub.execute_input":"2022-01-31T01:41:56.689189Z","iopub.status.idle":"2022-01-31T01:42:18.613365Z","shell.execute_reply.started":"2022-01-31T01:41:56.689142Z","shell.execute_reply":"2022-01-31T01:42:18.612301Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_df = pd.read_csv(\"../input/c/c/g-research-crypto-forecasting/asset_details.csv\", index_col=\"Asset_Name\")\nassets = list(asset_df.index)","metadata":{"_uuid":"6f03b5af-b4a2-482d-8992-977ab3364844","_cell_guid":"6e813325-7e2a-4a0b-8949-87d636ef6bf1","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:18.614900Z","iopub.execute_input":"2022-01-31T01:42:18.615202Z","iopub.status.idle":"2022-01-31T01:42:18.629900Z","shell.execute_reply.started":"2022-01-31T01:42:18.615158Z","shell.execute_reply":"2022-01-31T01:42:18.629005Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights = asset_df[\"Weight\"]\nweights = weights / weights.sum()","metadata":{"_uuid":"a06c1563-8239-4972-ad03-39ea0a24090b","_cell_guid":"2e45ce31-ed67-4e01-b364-53e3f877a3d9","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:18.631242Z","iopub.execute_input":"2022-01-31T01:42:18.631640Z","iopub.status.idle":"2022-01-31T01:42:18.644243Z","shell.execute_reply.started":"2022-01-31T01:42:18.631594Z","shell.execute_reply":"2022-01-31T01:42:18.643365Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"period1 = slice(\"2018-1-1\", \"2018-11-1\")  # 178560\n# weighted_ret.loc[period1].plot()\nperiod2 = slice(\"2019-1-1\", \"2019-7-1\")\n# weighted_ret.loc[period2].plot()\nperiod3 = slice(\"2019-8-1\", \"2019-12-1\")\n# weighted_ret.loc[period3].plot()\nperiod4 = slice(\"2020-1-1\", \"2021-5-1\")\n# weighted_ret.loc[period4].plot()\nperiod5 = slice(\"2021-6-1\", \"2021-10-1\")\n# weighted_ret.loc[period5].plot()\nperiod6 = slice(\"2021-10-1\", \"2022-1-25\")\n# weighted_ret.loc[period6].plot()","metadata":{"_uuid":"b39256a0-4b8e-4160-a070-13a46046df90","_cell_guid":"814ebddd-499b-40e4-85d4-752b2dab703a","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:18.646297Z","iopub.execute_input":"2022-01-31T01:42:18.646717Z","iopub.status.idle":"2022-01-31T01:42:18.655946Z","shell.execute_reply.started":"2022-01-31T01:42:18.646578Z","shell.execute_reply":"2022-01-31T01:42:18.654955Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocess the data\n# 1) delete the unused data\n# 2) forward fill\n# 3) normalize some features\nnew_dfs = {}\nfor asset_name in assets:\n    sub_df = df.loc[asset_name]\n    if asset_name == \"Maker\":\n        sub_df = sub_df.loc[\"2020-08-04\":]\n    elif asset_name == \"Monero\":\n        sub_df = sub_df.loc[\"2018-11-05\":]\n    elif asset_name == \"Stellar\":\n        sub_df = sub_df.loc[\"2018-07-14\":]\n#     sub_df = sub_df.loc[\"2020-1-1\":]\n    index = sub_df.index\n    sub_df = sub_df.fillna(method=\"ffill\").dropna()\n    if sub_df.size == 0:\n        raise ValueError()\n    for c in sub_df.columns:\n        if c in [\"Close\", \"High\", \"Low\", \"Open\", \"VWAP\"]:\n            sub_df[c] = sub_df[c] / sub_df[\"Close\"].iloc[0]\n        elif c in [\"Volume\", \"Count\"]:\n            sub_df[c] = sub_df[c] / sub_df[c].iloc[0]\n    new_dfs[asset_name] = sub_df\n    \ndf = pd.concat(new_dfs, axis=0, names=[\"Asset_ID\"])\ndel new_dfs","metadata":{"_uuid":"153fa955-60d4-435c-87b4-a0c921e270f1","_cell_guid":"7d2f2b6b-4a50-4cdb-a8ef-1a1dfed08785","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:18.658458Z","iopub.execute_input":"2022-01-31T01:42:18.659541Z","iopub.status.idle":"2022-01-31T01:42:41.084150Z","shell.execute_reply.started":"2022-01-31T01:42:18.659494Z","shell.execute_reply":"2022-01-31T01:42:41.083024Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_configs = {\n#  \"CHuWhiteCUMSUM_1200\": {\"func_name\": \"CHuWhiteCUMSUM\", \"timeperiod\": \"(1200, 3600)\"},\n#  \"CHuWhiteCUMSUM_240\": {\"func_name\": \"CHuWhiteCUMSUM\", \"timeperiod\": \"(240, 3600)\"},\n#  \"CHuWhiteCUMSUM_90\": {\"func_name\": \"CHuWhiteCUMSUM\", \"timeperiod\": \"(90, 3600)\"},\n#  \"HighLowVolatilityEstimator_240\": {\"func_name\": \"HighLowVolatilityEstimator\", \"timeperiod\": 240},\n#  \"HighLowVolatilityEstimator_90\": {\"func_name\": \"HighLowVolatilityEstimator\", \"timeperiod\": 90},\n#  \"HigLowLiquidityEstimator_240\": {\"func_name\": \"HigLowLiquidityEstimator\", \"timeperiod\": 240},\n#  \"HigLowLiquidityEstimator_90\": {\"func_name\": \"HigLowLiquidityEstimator\", \"timeperiod\": 90},\n    \n 'MFI_23': {'timeperiod': 23, 'correlation': 0, 'func_name': 'MFI'},\n 'DEMA_2_23': {'timeperiod': '(2, 23)', 'correlation': 0, 'func_name': 'DEMA'},\n 'RSI_16': {'timeperiod': 16, 'correlation': 0, 'func_name': 'RSI'},\n 'MACD_2_18': {'timeperiod': '(2, 18)', 'correlation': 0, 'func_name': 'MACD'},\n 'STD_7': {'timeperiod': 7, 'correlation': 0, 'func_name': 'STD'},\n 'EFFICIENCY_84': {'timeperiod': 84, 'correlation': 0,'func_name': 'EFFICIENCY'},\n 'MACD_HIST_2_163': {'timeperiod': '(2, 163)','correlation': 0,'func_name': 'MACD_HIST'},\n 'EVENT_MACROSS_1_83': {'timeperiod': '(1, 83)','correlation': 0,'func_name': 'EVENT_MACROSS'},\n 'RET_STD_90': {'timeperiod': 90, 'correlation': 0, 'func_name': 'RET_STD'},\n 'RSI_96': {'timeperiod': 96, 'correlation': 0, 'func_name': 'RSI'},\n 'TRIX_81': {'timeperiod': 81, 'correlation': 0, 'func_name': 'TRIX'},\n 'EI_20': {'timeperiod': 20, 'correlation': 0, 'func_name': 'EI'},\n 'RET_STD_23': {'timeperiod': 23, 'correlation': 0, 'func_name': 'RET_STD'},\n 'MFI_74': {'timeperiod': 74, 'correlation': 0, 'func_name': 'MFI'},\n 'EI_77': {'timeperiod': 77, 'correlation': 0, 'func_name': 'EI'},\n 'RET_15': {'timeperiod': 15, 'correlation': 0, 'func_name': 'RET'},\n 'STD_96': {'timeperiod': 96, 'correlation': 0, 'func_name': 'STD'},\n 'TRIX_4': {'timeperiod': 4, 'correlation': 0, 'func_name': 'TRIX'},\n 'EFFICIENCY_16': {'timeperiod': 16,'correlation': 0,'func_name': 'EFFICIENCY'},\n}","metadata":{"_uuid":"0f17c081-921e-422b-a294-4833c7e71122","_cell_guid":"13bbf2a2-e6fa-4144-a1a3-d595fd7f6e37","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:41.089052Z","iopub.execute_input":"2022-01-31T01:42:41.089912Z","iopub.status.idle":"2022-01-31T01:42:41.103883Z","shell.execute_reply.started":"2022-01-31T01:42:41.089861Z","shell.execute_reply":"2022-01-31T01:42:41.102172Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# store all data in a dictionary\ndv = {}\nfor feature in tqdm(df.columns):\n    dv[feature] = df.pivot(index=\"timestamp\", columns=\"Asset_ID\", values=feature)\n    dv[feature] = dv[feature].fillna(method=\"ffill\")","metadata":{"_uuid":"01028540-38b0-4401-a450-e790445797ba","_cell_guid":"3749d78d-db0b-4933-9fe5-71627788069a","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:42:41.105546Z","iopub.execute_input":"2022-01-31T01:42:41.106141Z","iopub.status.idle":"2022-01-31T01:46:40.026861Z","shell.execute_reply.started":"2022-01-31T01:42:41.106095Z","shell.execute_reply":"2022-01-31T01:46:40.025593Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"_uuid":"1626cbb1-e93b-492d-a27e-831a54066357","_cell_guid":"673cc46d-9fc8-4140-9942-4ea7ac5bb5f4","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:46:40.028857Z","iopub.execute_input":"2022-01-31T01:46:40.029430Z","iopub.status.idle":"2022-01-31T01:46:40.119226Z","shell.execute_reply.started":"2022-01-31T01:46:40.029385Z","shell.execute_reply":"2022-01-31T01:46:40.117941Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dv_features = get_features_vec(dv, all_configs)","metadata":{"_uuid":"491c44b2-135e-45e8-8c9b-80be13ecb709","_cell_guid":"798c9847-ac63-431e-aef7-e02cfbb8266f","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:46:40.120870Z","iopub.execute_input":"2022-01-31T01:46:40.121274Z","iopub.status.idle":"2022-01-31T01:47:46.664845Z","shell.execute_reply.started":"2022-01-31T01:46:40.121226Z","shell.execute_reply":"2022-01-31T01:47:46.663691Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = list(dv_features)","metadata":{"_uuid":"871da632-1e33-4576-81c8-bc719d9619e1","_cell_guid":"7178218c-7221-4f1c-a874-2139b7f68642","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:47:46.666942Z","iopub.execute_input":"2022-01-31T01:47:46.667284Z","iopub.status.idle":"2022-01-31T01:47:46.672200Z","shell.execute_reply.started":"2022-01-31T01:47:46.667240Z","shell.execute_reply":"2022-01-31T01:47:46.671170Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = dv[\"Target\"]\ndel dv","metadata":{"_uuid":"13605a83-036a-4ffe-8f5a-8b9a3db6a93f","_cell_guid":"d1e18b0f-f6c8-46f1-81c5-a09f8bbfc38a","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:47:46.673840Z","iopub.execute_input":"2022-01-31T01:47:46.674433Z","iopub.status.idle":"2022-01-31T01:47:46.762685Z","shell.execute_reply.started":"2022-01-31T01:47:46.674391Z","shell.execute_reply":"2022-01-31T01:47:46.761557Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc; gc.collect()","metadata":{"_uuid":"b40f8d90-96bf-4cfd-8acd-0f4210c64da0","_cell_guid":"783d9a43-ca83-4e85-bd3e-c8688e0b3130","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:47:46.765395Z","iopub.execute_input":"2022-01-31T01:47:46.766219Z","iopub.status.idle":"2022-01-31T01:47:46.926244Z","shell.execute_reply.started":"2022-01-31T01:47:46.766170Z","shell.execute_reply":"2022-01-31T01:47:46.925025Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"averaged_feature = [\"DEMA_2_23\", \"RET60_VWAP\", \"RSI_16\", \"RET_15\", \n                    \"RSI_96\", \"RET_STD_23\", \"RET_STD_90\", \"TRIX_4\",\n                   \"RET240_VWAP\", \"EFFICIENCY_84\", \"EFFICIENCY_16\", \n                    \"MACD_2_18\",\n                   ]","metadata":{"_uuid":"0fa28194-8f5e-438c-8729-8a3966e79944","_cell_guid":"7c072e3b-35b2-43bb-b6d6-7d85d2b1d377","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:47:46.928422Z","iopub.execute_input":"2022-01-31T01:47:46.929052Z","iopub.status.idle":"2022-01-31T01:47:46.937573Z","shell.execute_reply.started":"2022-01-31T01:47:46.928996Z","shell.execute_reply":"2022-01-31T01:47:46.936426Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feat in dv_features.keys():\n    path = os.path.join(\"{}.parquet\".format(feat))\n    dv_features[feat].to_parquet(path)\n#     if feat not in averaged_feature:\n#         del dv_features[feat]  # save memory","metadata":{"_uuid":"95ca3d79-b85e-4594-a663-b07b4176d189","_cell_guid":"0b266c11-b7d0-485e-bb37-7772893d9c00","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:47:46.939782Z","iopub.execute_input":"2022-01-31T01:47:46.940133Z","iopub.status.idle":"2022-01-31T01:48:46.260668Z","shell.execute_reply.started":"2022-01-31T01:47:46.940091Z","shell.execute_reply":"2022-01-31T01:48:46.259532Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rev = list(set(dv_features.keys()) - set(averaged_feature))\nfor feat in rev:\n    print(\"delete \", feat)\n    if feat in dv_features:\n        del dv_features[feat]  # save memory","metadata":{"_uuid":"d24f5f19-7719-4738-96eb-966713409aec","_cell_guid":"1752dd9a-750a-4afc-b14f-e73ccc0972e2","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:48:46.262612Z","iopub.execute_input":"2022-01-31T01:48:46.263256Z","iopub.status.idle":"2022-01-31T01:48:46.326024Z","shell.execute_reply.started":"2022-01-31T01:48:46.263209Z","shell.execute_reply":"2022-01-31T01:48:46.324854Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"_uuid":"7065c659-bc33-4056-95c5-d38430c05247","_cell_guid":"bf7d2801-1de8-4362-9183-19f84a2c543b","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:48:46.327842Z","iopub.execute_input":"2022-01-31T01:48:46.328156Z","iopub.status.idle":"2022-01-31T01:48:48.061884Z","shell.execute_reply.started":"2022-01-31T01:48:46.328111Z","shell.execute_reply":"2022-01-31T01:48:48.060752Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_original = False\nfor feat in tqdm(averaged_feature):\n    new_name = feat + \"_residualized\"\n    feature_names.append(new_name)\n    # dv_features[feat] 是一个(T, K)的矩阵，代表所有的asset的所有时间段的某个feature，然后减去mean\n    dv_features[new_name] = dv_features[feat].subtract(dv_features[feat].multiply(weights, axis=\"columns\").sum(axis=\"columns\"), axis=\"index\")\n    path = os.path.join(\"{}.parquet\".format(new_name))\n    dv_features[feat].to_parquet(path)\n    if drop_original:\n        feature_names.remove(feat)\n        \n    del dv_features[new_name]\n    if feat in dv_features:\n        del dv_features[feat]","metadata":{"_uuid":"25e6ee0e-b9c9-4d92-960b-aee791d08aac","_cell_guid":"6daa11fa-e528-431c-98fc-dbfc1a890550","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:48:48.063632Z","iopub.execute_input":"2022-01-31T01:48:48.067486Z","iopub.status.idle":"2022-01-31T01:49:12.584241Z","shell.execute_reply.started":"2022-01-31T01:48:48.067437Z","shell.execute_reply":"2022-01-31T01:49:12.583305Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_SCORE_DF = pd.DataFrame(index=asset_df.index, columns=[period1.start, period2.start, period3.start, period4.start, period5.start, period6.start])\nTRAIN_SCORE_DF = pd.DataFrame(index=asset_df.index, columns=[period1.start, period2.start, period3.start, period4.start, period5.start, period6.start])","metadata":{"_uuid":"4b0b9c15-d76b-4b95-97d2-cf85ca031eb1","_cell_guid":"0291f98c-dad4-4165-8707-e5885b736143","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:50:33.013057Z","iopub.execute_input":"2022-01-31T01:50:33.013425Z","iopub.status.idle":"2022-01-31T01:50:33.023815Z","shell.execute_reply.started":"2022-01-31T01:50:33.013355Z","shell.execute_reply":"2022-01-31T01:50:33.022720Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_SPLIT_DF = pd.DataFrame()\nFEAT_GAIN_DF = pd.DataFrame()","metadata":{"_uuid":"a3193d6b-8f32-415f-806c-4cb9a1d0ac7c","_cell_guid":"0731acab-b626-47bb-970e-bbec312a7b25","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:50:33.336113Z","iopub.execute_input":"2022-01-31T01:50:33.336370Z","iopub.status.idle":"2022-01-31T01:50:33.342160Z","shell.execute_reply.started":"2022-01-31T01:50:33.336342Z","shell.execute_reply":"2022-01-31T01:50:33.341025Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_score_for_one_symbol_new_cv(all_df, asset_id, feature_names, dry_run=False, model_params={}, dump_root=\"ckpt\"):\n    symbol_df = all_df\n    train_score_by_cv = [0] * N_SPLIT\n    test_score_by_cv = [0] * N_SPLIT\n    train_size_by_cv = [0] * N_SPLIT\n    test_size_by_cv = [0] * N_SPLIT\n    test_period_by_cv = [0] * N_SPLIT\n    test_type_by_cv = [0] * N_SPLIT\n    iter_by_cv = [0] * N_SPLIT\n    df_proc = symbol_df[feature_names]\n    bulls = [period4, period2]\n    bears = [period3]\n    neutral = [period1, period5, period6]\n    for i, period in enumerate([period1, period2, period3, period4, period5, period6]):\n        print(period)\n        train_features, train_target = df_proc.loc[period], symbol_df[\"Target\"].loc[period]\n        test_features, test_target = df_proc.loc[period], symbol_df[\"Target\"].loc[period]\n        if test_features.size == 0:\n            continue\n        part1 = pd.Timestamp(period.start) - pd.Timedelta(\"30D\")\n        part2 = pd.Timestamp(period.stop) + pd.Timedelta(\"30D\")\n        dfs = []\n        targets = []\n        _df1 = df_proc.loc[:part1]\n        _df2 = df_proc.loc[part2:]\n        if _df1.size > 0:\n            dfs.append(_df1)\n            targets.append(symbol_df[\"Target\"].loc[:part1])\n        if _df2.size > 0:\n            dfs.append(_df2)\n            targets.append(symbol_df[\"Target\"].loc[part2:])\n        if len(dfs) == 2:\n            train_features = pd.concat(dfs)\n            train_target = pd.concat(targets)\n        elif len(dfs) == 1:\n            train_features = dfs[0]\n            train_target = targets[0]\n        else:\n            continue\n        train_size = len(train_features)\n        test_size = len(test_features)\n        train_features = train_features.replace([np.inf, -np.inf], np.nan)\n        test_features = test_features.replace([np.inf, -np.inf], np.nan)\n        train_nan_mask = train_features.isnull().any(axis=1)\n        test_nan_mask = test_features.isnull().any(axis=1)\n        train_features, train_target = train_features.loc[~train_nan_mask], train_target.loc[~train_nan_mask]\n        test_features, test_target = test_features.loc[~test_nan_mask], test_target.loc[~test_nan_mask]\n        \n        train_set = lgb.Dataset(train_features, label=train_target, feature_name=feature_names)\n        test_set = lgb.Dataset(test_features, label=test_target, feature_name=feature_names)\n        # continuous\n        \n        assert len(train_features) == len(train_target), \"{}_{}\".format(len(train_features), len(train_target))\n        assert len(test_features) == len(test_target), \"{}_{}\".format(len(train_features), len(train_target))\n        booster = lgb.train(train_set=train_set, params=model_params, valid_sets=[test_set], feval=pearson_eval)\n        corr_train = validate_one_symble(booster, train_features, train_target)\n        corr_test = validate_one_symble(booster, test_features, test_target)\n        TEST_SCORE_DF.loc[asset_id, period.start] = float(corr_test)\n        TRAIN_SCORE_DF.loc[asset_id, period.start] = float(corr_train)\n        for name in feature_names:\n            FEAT_SPLIT_DF.loc[name, \"{}_{}\".format(asset_id, period.start)] = dict(zip(feature_names, booster.feature_importance(\"split\")))[name]\n            FEAT_GAIN_DF.loc[name, \"{}_{}\".format(asset_id, period.start)] = dict(zip(feature_names, booster.feature_importance(\"gain\")))[name]\n\n        train_score_by_cv[i] = float(corr_train)\n        test_score_by_cv[i] = float(corr_test)\n        train_size_by_cv[i] = int(train_size)\n        test_size_by_cv[i] = int(test_size)\n        test_period_by_cv[i] = [period.start, period.stop]\n        if period in bulls:\n            test_type_by_cv[i] = \"bull\"\n        elif period in bears:\n            test_type_by_cv[i] = \"bear\"\n        else:\n            test_type_by_cv[i] = \"neutral\"\n        iter_by_cv[i] = booster.best_iteration\n        str_path = os.path.join(os.getcwd(), dump_root, asset_id, str(i))\n        os.makedirs(str_path, exist_ok=True)\n        model_str = booster.model_to_string()\n        with open(os.path.join(str_path, \"lgb.ckpt\"), \"w\") as f:\n            f.write(model_str)\n        with open(os.path.join(str_path, \"used_features.pickle\"), \"wb\") as f:\n            pickle.dump(feature_names, f)\n        if dry_run:\n            break\n    avg_train_score = sum(train_score_by_cv) / N_SPLIT\n    avg_test_score = sum(test_score_by_cv) / N_SPLIT\n    best_iteration = booster.best_iteration\n    meta = {\n            \"train_score\": train_score_by_cv,\n            \"test_score\": test_score_by_cv,\n            \"train_size_by_cv\": train_size_by_cv,\n            \"test_size_by_cv\": test_size_by_cv,\n            \"test_type_by_cv\": test_type_by_cv,\n            \"test_period_by_cv\": test_period_by_cv,\n            \"model_params\": model_params,\n            \"avg_train_score\": avg_train_score,\n            \"avg_test_score\": avg_test_score,\n            \"iter_by_cv\": iter_by_cv\n        }\n        \n    meta_path = os.path.join(os.getcwd(), dump_root, asset_id, \"lgb_meta.json\")\n    with open(meta_path, \"w\") as f:\n        f.write(json.dumps(meta, indent=2))\n    return avg_train_score, avg_test_score, meta","metadata":{"_uuid":"cc5c9a50-83dd-4e8b-a624-1a820f13944c","_cell_guid":"0d2f7cfe-9194-4d71-8b26-2210ff9cd76f","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:53:26.946609Z","iopub.execute_input":"2022-01-31T01:53:26.947159Z","iopub.status.idle":"2022-01-31T01:53:26.981817Z","shell.execute_reply.started":"2022-01-31T01:53:26.947124Z","shell.execute_reply":"2022-01-31T01:53:26.980406Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(feature_names)","metadata":{"_uuid":"65331ffe-69b5-472e-ba82-0be2e53cf233","_cell_guid":"fc2bf505-99e3-437b-a789-dafcba0f313d","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:50:38.571365Z","iopub.execute_input":"2022-01-31T01:50:38.571666Z","iopub.status.idle":"2022-01-31T01:50:38.578952Z","shell.execute_reply.started":"2022-01-31T01:50:38.571615Z","shell.execute_reply":"2022-01-31T01:50:38.577702Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(feature_names))","metadata":{"_uuid":"2ae242e9-dff1-437f-9eed-9444ed861c28","_cell_guid":"d1e41a45-28e1-4491-b64d-63395060f3b3","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:50:45.732099Z","iopub.execute_input":"2022-01-31T01:50:45.732636Z","iopub.status.idle":"2022-01-31T01:50:45.738708Z","shell.execute_reply.started":"2022-01-31T01:50:45.732589Z","shell.execute_reply":"2022-01-31T01:50:45.737613Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_score_by_symbol = {}\ntest_score_by_symbol = {}\n\nfor asset_id in assets:\n    df = {\"Target\": target[asset_id]}\n    for feat in tqdm(feature_names):\n        feature_df = pd.read_parquet(os.path.join(\"{}.parquet\".format(feat)), columns=[asset_id])[asset_id]\n        df[feat] = feature_df\n        del feature_df\n    df = pd.concat(df, axis=1)\n    df.index = pd.to_datetime(df.index, unit='s')\n    df.dropna(subset=feature_names+[\"Target\"], inplace=True)\n    print(asset_id + \"\\n***\")\n    train_score, test_score, meta = get_score_for_one_symbol_new_cv(df, asset_id, feature_names, dry_run=True, model_params=MODEL_PARAMS, dump_root=CKPT)\n    train_score_by_symbol[asset_id] = train_score\n    test_score_by_symbol[asset_id] = test_score\n    \n    print(meta)\n    print(\"\\n\")","metadata":{"_uuid":"2c0d0fcb-1fb1-460f-b7f5-cb60b5308cd4","_cell_guid":"b5aa58c6-87cf-4173-9fc7-21abd1dce1c0","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:55:24.697615Z","iopub.execute_input":"2022-01-31T01:55:24.697977Z","iopub.status.idle":"2022-01-31T01:58:03.695467Z","shell.execute_reply.started":"2022-01-31T01:55:24.697940Z","shell.execute_reply":"2022-01-31T01:58:03.693573Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SCORE_DF","metadata":{"_uuid":"c3a7a5ee-1892-455b-b103-f0722f821f94","_cell_guid":"fe5ceaed-53fc-41cc-8f07-0b41a2191dd5","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:58:09.063188Z","iopub.execute_input":"2022-01-31T01:58:09.063977Z","iopub.status.idle":"2022-01-31T01:58:09.094544Z","shell.execute_reply.started":"2022-01-31T01:58:09.063920Z","shell.execute_reply":"2022-01-31T01:58:09.093369Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TEST_SCORE_DF","metadata":{"_uuid":"7cd5584c-790f-49ef-9ea4-89cae2679058","_cell_guid":"bd58237d-3bed-40be-bf3d-d939d7c215f0","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:58:12.013687Z","iopub.execute_input":"2022-01-31T01:58:12.014140Z","iopub.status.idle":"2022-01-31T01:58:12.032037Z","shell.execute_reply.started":"2022-01-31T01:58:12.014101Z","shell.execute_reply":"2022-01-31T01:58:12.030612Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_SCORE_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"train_score_df.csv\"))\nTEST_SCORE_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"test_score_df.csv\"))\nFEAT_SPLIT_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"FEAT_SPLIT_DF.csv\"))\nFEAT_GAIN_DF.to_csv(os.path.join(os.getcwd(), CKPT, \"FEAT_GAIN_DF.csv\"))","metadata":{"_uuid":"122c60de-5171-4132-afe6-19edcc2aea1b","_cell_guid":"bff6eb0b-123b-45db-9d96-12209d5e2433","collapsed":false,"execution":{"iopub.status.busy":"2022-01-31T01:58:14.190973Z","iopub.execute_input":"2022-01-31T01:58:14.192151Z","iopub.status.idle":"2022-01-31T01:58:14.208867Z","shell.execute_reply.started":"2022-01-31T01:58:14.192104Z","shell.execute_reply":"2022-01-31T01:58:14.207712Z"},"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_train_score = sum([score * weights[s] for s, score in train_score_by_symbol.items()])\nfinal_test_score = sum([score * weights[s] for s, score in test_score_by_symbol.items()])\nprint(\"avg. model score on train: {:.4f}\".format(final_train_score))\nprint(\"avg. model score on test: {:.4f}\".format(final_test_score))","metadata":{"_uuid":"92276308-1528-4077-95d5-a9314899c7c3","_cell_guid":"fe06d191-7087-4fd3-ae23-6f8e1ab5ddcc","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_by_symbol = pd.DataFrame({\"train_score\": train_score_by_symbol, \"test_score\": test_score_by_symbol}).sort_values(by=\"train_score\")","metadata":{"_uuid":"2d65bdbc-1876-487c-a3f1-26f503d53833","_cell_guid":"43e46d74-51e4-4c2b-97aa-48a68a153473","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read baseline model\nnew_meta_root = CKPT\nold_scores = pd.read_csv(os.path.join(\"../input/baselineresult/ckpt\", \"test_score_df.csv\"), index_col=\"Asset_Name\")\nnew_scores = pd.read_csv(os.path.join(CKPT, \"test_score_df.csv\"), index_col=\"Asset_Name\")\nscore_dif = new_scores - old_scores\n# show_dif = pd.DataFrame(index=new_scores.index, columns=new_scores.columns, dtype=float)\n# for i in new_scores.index:\n#     for j in new_scores.columns:\n#         diff = new_scores.loc[i, j] - old_scores.loc[i, j]\n#         if diff > 0:\n#             show_dif.loc[i,j] = \"{:.2%}(+{:.2%})\".format(new_scores.loc[i, j], diff)\n#         else:\n#             show_dif.loc[i,j] = \"{:.2%}({:.2%})\".format(new_scores.loc[i, j], diff)\n# show_dif","metadata":{"_uuid":"b64c19f7-81c3-4874-94fb-a361bac06710","_cell_guid":"42520108-fd4d-4e8c-a074-9cb9dfc38e0b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"new fold result\")\nprint(new_scores.multiply(weights, axis=\"index\").sum(axis=0))\nprint(\"baseline fold result\")\nprint(old_scores.multiply(weights, axis=\"index\").sum(axis=0))\n\nprint(\"new fold std result\")\nprint(new_scores.multiply(weights, axis=\"index\").sum(axis=0).std())\nprint(\"baseline fold std result\")\nprint(old_scores.multiply(weights, axis=\"index\").sum(axis=0).std())\n\nprint(\"new fold mean result\")\nprint(new_scores.multiply(weights, axis=\"index\").sum(axis=0).mean())\nprint(\"baseline fold mean result\")\nprint(old_scores.multiply(weights, axis=\"index\").sum(axis=0).mean())","metadata":{"_uuid":"03b2eeff-4adc-4003-8c65-89cab1888e32","_cell_guid":"74b2ea37-68d8-47df-86b4-3ec321fcc585","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.stats import ttest_1samp\nscore_dif = score_dif.astype(float)\ndif = np.ravel(score_dif.values)\ndif = dif[~np.isnan(dif)]\nttest_1samp(dif, 0)","metadata":{"_uuid":"26a96216-7a1e-4cb2-8b75-a1ba2f49c2ab","_cell_guid":"5555fbfe-0c56-4965-830c-273e4958c1e1","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_dif.mean(axis=1)","metadata":{"_uuid":"9b17a78a-968d-4295-ac6b-6d42373aa11f","_cell_guid":"9802d44a-39a4-4e2f-9015-0c63595bfa48","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score_dif.mean(axis=0)","metadata":{"_uuid":"d94c0138-0c3e-4a33-8813-180891edd135","_cell_guid":"1ccc7c7b-aaa7-4424-bad7-03a6dc952f8e","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.nanmean(score_dif)","metadata":{"_uuid":"19a2585d-8fc8-4bc1-a010-b14d31c6a5e7","_cell_guid":"9202e75d-637f-43ba-83b2-124333742119","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_GAIN_DF.rank().std(axis=1).sort_values()","metadata":{"_uuid":"f79e0d09-b757-461d-909b-fe7ce9ff85f4","_cell_guid":"777fa7a0-9a42-4200-a8ed-e5dcc07c4490","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_SPLIT_DF.rank().std(axis=1).sort_values()","metadata":{"_uuid":"20838972-5721-4146-858d-fc0e4b2212e3","_cell_guid":"4b121c67-dd21-475c-930c-0c688fec7ca0","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_GAIN_DF.rank().mean(axis=1).sort_values()","metadata":{"_uuid":"c996c515-061e-4f96-8609-eb170a18b7c6","_cell_guid":"63e8d74f-b016-40b5-b844-2588ef4fd790","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FEAT_SPLIT_DF.rank().mean(axis=1).sort_values()","metadata":{"_uuid":"429edb4c-1f2c-45f9-bc3a-9beb6368ff66","_cell_guid":"9fa1c6a4-e622-4858-89df-f45aa05394e7","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}