{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🪙 Examination of features using LGBM\n---\n\nThe objective of this notebook is to explore effective features using the importance of LGBM features..<br>\nFirst, training by LGBM. Then check the importance of each feature.<br>\nこのノートブックでは、LGBMのFeature Importanceを可視化して、効果的な特徴量を探索しています。\nまず、LGBMで学習をし、その後、Feature Importanceを確認します。\n---\n![](https://storage.googleapis.com/kaggle-competitions/kaggle/30894/logos/header.png)\n---\nReferences: <br>\n1. [Tutorial to the G-Research Crypto Competition](https://www.kaggle.com/cstein06/tutorial-to-the-g-research-crypto-competition)<br>\n2. [🪙💲 G-Research- Starter LGBM Pipeline](https://www.kaggle.com/julian3833/g-research-starter-lgbm-pipeline)<br>\n3. [Feature Engineering: Rolling Aggregations](https://www.kaggle.com/yamqwe/feature-engineering-rolling-aggregations?scriptVersionId=81111556)<br>\n\nPlease if this kernel is useful, please upvote !! (宜しければUpvoteお願いします！)","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\n\nimport os\nfrom tqdm import tqdm\nimport traceback\nimport random\nimport pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nfrom scipy.stats import pearsonr\nfrom sklearn.metrics import r2_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\nplt.rcParams.update({'font.size': 18})\nplt.style.use('fivethirtyeight')\npd.set_option('display.max_columns', 150)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-05T11:15:20.080441Z","iopub.execute_input":"2022-01-05T11:15:20.080807Z","iopub.status.idle":"2022-01-05T11:15:22.127182Z","shell.execute_reply.started":"2022-01-05T11:15:20.080693Z","shell.execute_reply":"2022-01-05T11:15:22.125926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"DEVICE = \"GPU\" #or \"TPU\"\n\nSEED = 42\n\n# CV PARAMS\nFOLDS = 5\nGROUP_GAP = 130\nMAX_TEST_GROUP_SIZE = 180\nMAX_TRAIN_GROUP_SIZE = 280\n\n# LOAD STRICT? YES=1 NO=0 | see: https://www.kaggle.com/julian3833/proposal-for-a-meaningful-lb-strict-lgbm\nLOAD_STRICT = True\n\n# WHICH YEARS TO INCLUDE? YES=1 NO=0\nINC2021 = 0\nINC2020 = 0\nINC2019 = 0\nINC2018 = 0\nINC2017 = 0\nINCCOMP = 1\nINCSUPP = 0\n\n# BATCH SIZE AND EPOCHS (for CNN)\n# BATCH_SIZES = [2048] * FOLDS # [2048, 2048, 2048, 2048, 2048]\n# EPOCHS = [1] * FOLDS         # [1, 1, 1, 1, 1]\n\n# FEAT_WIN_SIZE = 10 # Window size for feature extraction","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:22.128907Z","iopub.execute_input":"2022-01-05T11:15:22.129140Z","iopub.status.idle":"2022-01-05T11:15:22.134730Z","shell.execute_reply.started":"2022-01-05T11:15:22.129111Z","shell.execute_reply":"2022-01-05T11:15:22.133808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nfix_all_seeds(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:22.136139Z","iopub.execute_input":"2022-01-05T11:15:22.136366Z","iopub.status.idle":"2022-01-05T11:15:22.147086Z","shell.execute_reply.started":"2022-01-05T11:15:22.136330Z","shell.execute_reply":"2022-01-05T11:15:22.145975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{}},{"cell_type":"code","source":"import datatable as dt\nextra_data_files = {0: '../input/cryptocurrency-extra-data-binance-coin', \n                    1: '../input/cryptocurrency-extra-data-bitcoin', \n                    2: '../input/cryptocurrency-extra-data-bitcoin-cash', \n                    3: '../input/cryptocurrency-extra-data-cardano', \n                    4: '../input/cryptocurrency-extra-data-dogecoin', \n                    5: '../input/cryptocurrency-extra-data-eos-io', \n                    6: '../input/cryptocurrency-extra-data-ethereum', \n                    7: '../input/cryptocurrency-extra-data-ethereum-classic', \n                    8: '../input/cryptocurrency-extra-data-iota', \n                    9: '../input/cryptocurrency-extra-data-litecoin', \n                    11: '../input/cryptocurrency-extra-data-monero', \n                    10: '../input/cryptocurrency-extra-data-maker', \n                    12: '../input/cryptocurrency-extra-data-stellar', \n                    13: '../input/cryptocurrency-extra-data-tron'\n                    }\n\n# Uncomment to load the original csv [slower]\n# orig_df_train = pd.read_csv(data_path + 'train.csv') \n# supp_df_train = pd.read_csv(data_path + 'supplemental_train.csv')\n# df_asset_details = pd.read_csv(data_path  + 'asset_details.csv').sort_values(\"Asset_ID\")\n\norig_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_train.jay').to_pandas()\ndf_asset_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nsupp_df_train = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_supplemental_train.jay').to_pandas()\nassets_details = dt.fread('../input/cryptocurrency-extra-data-binance-coin/orig_asset_details.jay').to_pandas()\nasset_weight_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Weight'].tolist()[idx] for idx in range(len(assets_details))}\nasset_name_dict = {assets_details['Asset_ID'].tolist()[idx]: assets_details['Asset_Name'].tolist()[idx] for idx in range(len(assets_details))}","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:22.149046Z","iopub.execute_input":"2022-01-05T11:15:22.149445Z","iopub.status.idle":"2022-01-05T11:15:44.967775Z","shell.execute_reply.started":"2022-01-05T11:15:22.149413Z","shell.execute_reply":"2022-01-05T11:15:44.966908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_training_data_for_asset(asset_id, load_jay = True):\n    \"\"\"Asset_idごとのトレーニングデータを生成する。\n\n    Args:\n        asset_id (int): アセットID\n        load_jay (bool, optional): jayデータをロードするか. Defaults to True.\n\n    Returns:\n        DataFrame: AssetIdのデータフレーム\n    \"\"\"\n    dfs = []\n    \n    # オリジナルから、AssetIDの該当するデータを読み込む。\n    if INCCOMP: \n        dfs.append(orig_df_train[orig_df_train[\"Asset_ID\"] == asset_id].copy())\n    if INCSUPP: \n        dfs.append(supp_df_train[supp_df_train[\"Asset_ID\"] == asset_id].copy())\n    \n    if load_jay:\n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): \n            dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.jay').to_pandas())\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): \n            dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.jay').to_pandas())\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): \n            dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.jay').to_pandas())\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): \n            dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.jay').to_pandas())\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): \n            dfs.append(dt.fread(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.jay').to_pandas())\n    else:\n        # CSVで読み込む\n        if INC2017 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'): \n            dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2017) + '.csv'))\n        if INC2018 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'): \n            dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2018) + '.csv'))\n        if INC2019 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'): \n            dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2019) + '.csv'))\n        if INC2020 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'): \n            dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2020) + '.csv'))\n        if INC2021 and os.path.exists(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'): \n            dfs.append(pd.read_csv(extra_data_files[asset_id] + '/full_data__' + str(asset_id) + '__' + str(2021) + '.csv'))\n    \n    # dfsリストに格納していたDFをCONCAT        \n    df = pd.concat(dfs, axis = 0) if len(dfs) > 1 else dfs[0]\n    df['date'] = pd.to_datetime(df['timestamp'], unit = 's')\n    if LOAD_STRICT: \n        df = df.loc[df['date'] < \"2021-06-13 00:00:00\"]\n    df = df.sort_values('date')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:44.969013Z","iopub.execute_input":"2022-01-05T11:15:44.969249Z","iopub.status.idle":"2022-01-05T11:15:44.987420Z","shell.execute_reply.started":"2022-01-05T11:15:44.969219Z","shell.execute_reply":"2022-01-05T11:15:44.986457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_for_all_assets():\n    \"\"\"各AssetIDのデータフレームをload_training_data_for_assetでロードした後に\n        CONCATし、1つのデータフレームにする。\n\n    Returns:\n        DataFrame: 全AssetIDを含むDATAFRAME\n    \"\"\"\n    dfs = []\n    for asset_id in list(extra_data_files.keys()): \n        dfs.append(load_training_data_for_asset(asset_id))\n    return pd.concat(dfs)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:44.989053Z","iopub.execute_input":"2022-01-05T11:15:44.989401Z","iopub.status.idle":"2022-01-05T11:15:45.007075Z","shell.execute_reply.started":"2022-01-05T11:15:44.989372Z","shell.execute_reply":"2022-01-05T11:15:45.006233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"code","source":"#####################################################################\n# base stats\n#####################################################################\ndef mean(x):\n    return np.mean(x)\ndef length(x):\n    return len(x)\ndef standard_deviation(x):\n    return np.std(x)\n\ndef variation_coefficient(x):\n    mean = np.mean(x)\n    if mean != 0:\n        return np.std(x) / mean\n    else:\n        return np.nan\n\ndef variance(x):\n    return np.var(x)\n\ndef skewness(x):\n    if not isinstance(x, pd.Series):\n        x = pd.Series(x)\n    return pd.Series.skew(x)\n\ndef kurtosis(x):\n    if not isinstance(x, pd.Series):\n        x = pd.Series(x)\n    return pd.Series.kurtosis(x)\n\n#####################################################################\n# higher_order_stats\n#####################################################################\ndef abs_energy(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    return np.dot(x, x)\n\ndef root_mean_square(x):\n    return np.sqrt(np.mean(np.square(x))) if len(x) > 0 else np.NaN\n\ndef sum_values(x):\n    if len(x) == 0:\n        return 0\n    return np.sum(x)\n\n# def realized_volatility(series_log_return):\n#     return np.sqrt(np.sum(series_log_return**2))\ndef realized_volatility(series):\n    return np.sqrt(np.sum(series**2))\n\ndef realized_abs_skew(series):\n    return np.power(np.abs(np.sum(series**3)),1/3)\n\ndef realized_skew(series):\n    return np.sign(np.sum(series**3))*np.power(np.abs(np.sum(series**3)),1/3)\n\ndef realized_vol_skew(series):\n    return np.power(np.abs(np.sum(series**6)),1/6)\n\ndef realized_quarticity(series):\n    return np.power(np.sum(series**4),1/4)\n#####################################################################\n# min_median_max\n#####################################################################\ndef minimum(x):\n    return np.min(x)\ndef median(x):\n    return np.median(x)\ndef maximum(x):\n    return np.max(x)\n#####################################################################\n# additional_quantile\n#####################################################################\nquantile_01 = lambda x: quantile(x,0.1)\nquantile_01.__name__ = 'quantile_01'\n\nquantile_025 = lambda x: quantile(x,0.25)\nquantile_025.__name__ = 'quantile_025'\n\nquantile_075 = lambda x: quantile(x,0.75)\nquantile_075.__name__ = 'quantile_075'\n\nquantile_09 = lambda x: quantile(x,0.9)\nquantile_09.__name__ = 'quantile_09'\n#####################################################################\n# other_minmax\n#####################################################################\ndef absolute_maximum(x):\n    return np.max(np.absolute(x)) if len(x) > 0 else np.NaN\n\ndef max_over_min(series):\n    if len(series)<2:\n        return 0\n    if np.min(series) == 0:\n        return np.nan\n    return np.max(series)/np.min(series)\n\ndef max_over_min_sq(series):\n    if len(series)<2:\n        return 0\n    if np.min(series) == 0:\n        return np.nan\n    return np.square(np.max(series)/np.min(series))\n\n#####################################################################\n# minmax_positions\n#####################################################################\ndef last_location_of_maximum(x):\n    x = np.asarray(x)\n    return 1.0 - np.argmax(x[::-1]) / len(x) if len(x) > 0 else np.NaN\n\ndef first_location_of_maximum(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    return np.argmax(x) / len(x) if len(x) > 0 else np.NaN\n\ndef last_location_of_minimum(x):\n    x = np.asarray(x)\n    return 1.0 - np.argmin(x[::-1]) / len(x) if len(x) > 0 else np.NaN\n\ndef first_location_of_minimum(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    return np.argmin(x) / len(x) if len(x) > 0 else np.NaN\n\n#####################################################################\n# peaks\n#####################################################################\nnumber_peaks_2 = lambda x: number_peaks(x,2)\nnumber_peaks_2.__name__ = 'number_peaks_2'\n\nmean_n_absolute_max_2 = lambda x: mean_n_absolute_max(x,2)\nmean_n_absolute_max_2.__name__ = 'mean_n_absolute_max_2'\n\nnumber_peaks_5 = lambda x: number_peaks(x,5)\nnumber_peaks_5.__name__ = 'number_peaks_5'\n\nmean_n_absolute_max_5 = lambda x: mean_n_absolute_max(x,5)\nmean_n_absolute_max_5.__name__ = 'mean_n_absolute_max_5'\n\nnumber_peaks_10 = lambda x: number_peaks(x,10)\nnumber_peaks_10.__name__ = 'number_peaks_10'\n\nmean_n_absolute_max_10 = lambda x: mean_n_absolute_max(x,10)\nmean_n_absolute_max_10.__name__ = 'mean_n_absolute_max_10'\n\n#####################################################################\n# counts\n#####################################################################\ndef count_unique(series):\n    return len(np.unique(series))\n\ndef count(series):\n    return series.size\n\ncount_above_0 = lambda x: count_above(x,0)\ncount_above_0.__name__ = 'count_above_0'\n\ncount_below_0 = lambda x: count_below(x,0)\ncount_below_0.__name__ = 'count_below_0'\n\nvalue_count_0 = lambda x: value_count(x,0)\nvalue_count_0.__name__ = 'value_count_0'\n\ncount_near_0 = lambda x: range_count(x,-0.00001,0.00001)\ncount_near_0.__name__ = 'count_near_0_0'\n\n#####################################################################\n# reoccurring values\n#####################################################################\ndef count_above_mean(x):\n    m = np.mean(x)\n    return np.where(x > m)[0].size\n\ndef count_below_mean(x):\n    m = np.mean(x)\n    return np.where(x < m)[0].size\n\n# Test non-consecutive non-reoccuring values ?\ndef percentage_of_reoccurring_values_to_all_values(x):\n    if len(x) == 0:\n        return np.nan\n    unique, counts = np.unique(x, return_counts=True)\n    if counts.shape[0] == 0:\n        return 0\n    return np.sum(counts > 1) / float(counts.shape[0])\n\ndef percentage_of_reoccurring_datapoints_to_all_datapoints(x):\n    if len(x) == 0:\n        return np.nan\n    if not isinstance(x, pd.Series):\n        x = pd.Series(x)\n    value_counts = x.value_counts()\n    reoccuring_values = value_counts[value_counts > 1].sum()\n    if np.isnan(reoccuring_values):\n        return 0\n\n    return reoccuring_values / x.size\n\ndef sum_of_reoccurring_values(x):\n    unique, counts = np.unique(x, return_counts=True)\n    counts[counts < 2] = 0\n    counts[counts > 1] = 1\n    return np.sum(counts * unique)\n\ndef sum_of_reoccurring_data_points(x):\n    unique, counts = np.unique(x, return_counts=True)\n    counts[counts < 2] = 0\n    return np.sum(counts * unique)\n\ndef ratio_value_number_to_time_series_length(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    if x.size == 0:\n        return np.nan\n\n    return np.unique(x).size / x.size\n\n#####################################################################\n# count suplicates\n#####################################################################\ndef count_duplicate_max(x):\n    return np.sum(x == np.max(x))\n\ndef count_duplicate_min(x):\n    return np.sum(x == np.min(x))\n\ndef count_duplicate(x):\n    return x.size - np.unique(x).size\n\n#####################################################################\n# valiations\n#####################################################################\ndef mean_abs_change(x):\n    return np.mean(np.abs(np.diff(x)))\n\ndef mean_change(x):\n    x = np.asarray(x)\n    return (x[-1] - x[0]) / (len(x) - 1) if len(x) > 1 else np.NaN\n\ndef mean_second_derivative_central(x):\n    x = np.asarray(x)\n    return (x[-1] - x[-2] - x[1] + x[0]) / (2 * (len(x) - 2)) if len(x) > 2 else np.NaN\n\ndef absolute_sum_of_changes(x):\n    return np.sum(np.abs(np.diff(x)))\n\nnumber_crossing_0 = lambda x: number_crossing_m(x,0)\nnumber_crossing_0.__name__ = 'number_crossing_0'\n\n#####################################################################\n# ranges\n#####################################################################\ndef variance_std_ratio(x):\n    y = np.var(x)\n    if y != 0:\n        return y/np.sqrt(y)\n    else:\n        return np.nan\n\ndef ratio_beyond_r_sigma(x, r):\n    if x.size == 0:\n        return np.nan\n    else:\n        return np.sum(np.abs(x - np.mean(x)) > r * np.asarray(np.std(x))) / x.size\n\nratio_beyond_01_sigma = lambda x: ratio_beyond_r_sigma(x,0.1)\nratio_beyond_01_sigma.__name__ = 'ratio_beyond_01_sigma'\n\nratio_beyond_02_sigma = lambda x: ratio_beyond_r_sigma(x,0.2)\nratio_beyond_02_sigma.__name__ = 'ratio_beyond_02_sigma'\n\nratio_beyond_03_sigma = lambda x: ratio_beyond_r_sigma(x,0.3)\nratio_beyond_03_sigma.__name__ = 'ratio_beyond_03_sigma'\n\ndef large_standard_deviation(x):\n    if (np.max(x)-np.min(x)) == 0:\n        return np.nan\n    else:\n        return np.std(x)/(np.max(x)-np.min(x))\n    \ndef range_ratio(x):\n    mean_median_difference = np.abs(np.mean(x) - np.median(x))\n    max_min_difference = np.max(x) - np.min(x)\n    if max_min_difference == 0:\n        return np.nan\n    else:\n        return mean_median_difference / max_min_difference\n    \n#####################################################################\n# get first fn\n#####################################################################\nget_first = lambda x: x.iloc[0]\nget_first.__name__ = 'get_first'\n\nget_last = lambda x: x.iloc[-1]\nget_last.__name__ = 'get_last'\n\n#####################################################################\n# draw functions\n#####################################################################\n#drawdons functions are mine\ndef maximum_drawdown(series):\n    series = np.asarray(series)\n    if len(series)<2:\n        return 0\n    k = series[np.argmax(np.maximum.accumulate(series) - series)]\n    i = np.argmax(np.maximum.accumulate(series) - series)\n    if len(series[:i])<1:\n        return np.NaN\n    else:\n        j = np.max(series[:i])\n    return j-k\n\ndef maximum_drawup(series):\n    series = np.asarray(series)\n    if len(series)<2:\n        return 0\n    \n    series = - series\n    k = series[np.argmax(np.maximum.accumulate(series) - series)]\n    i = np.argmax(np.maximum.accumulate(series) - series)\n    if len(series[:i])<1:\n        return np.NaN\n    else:\n        j = np.max(series[:i])\n    return j-k\n\ndef drawdown_duration(series):\n    series = np.asarray(series)\n    if len(series)<2:\n        return 0\n\n    k = np.argmax(np.maximum.accumulate(series) - series)\n    i = np.argmax(np.maximum.accumulate(series) - series)\n    if len(series[:i]) == 0:\n        j=k\n    else:\n        j = np.argmax(series[:i])\n    return k-j\n\ndef drawup_duration(series):\n    series = np.asarray(series)\n    if len(series)<2:\n        return 0\n\n    series=-series\n    k = np.argmax(np.maximum.accumulate(series) - series)\n    i = np.argmax(np.maximum.accumulate(series) - series)\n    if len(series[:i]) == 0:\n        j=k\n    else:\n        j = np.argmax(series[:i])\n    return k-j\n\n#####################################################################\n# Others\n#####################################################################\ndef _roll(a, shift):\n    \"\"\" Roll 1D array elements. Improves the performance of numpy.roll()\"\"\"\n\n    if not isinstance(a, np.ndarray):\n        # 型が合っていなかったら直す。\n        a = np.asarray(a)\n    idx = shift % len(a)\n    return np.concatenate([a[-idx:], a[:-idx]])\n\ndef _get_length_sequences_where(x):\n    \"\"\" This method calculates the length of all sub-sequences where the array x is either True or 1. \"\"\"\n    if len(x) == 0:\n        return [0]\n    else:\n        res = [len(list(group)) for value, group in itertools.groupby(x) if value == 1]\n        return res if len(res) > 0 else [0]\n\ndef _aggregate_on_chunks(x, f_agg, chunk_len):\n    \"\"\"Takes the time series x and constructs a lower sampled version of it by applying the aggregation function f_agg on\n    consecutive chunks of length chunk_len\"\"\"\n    \n    return [\n        getattr(x[i * chunk_len : (i + 1) * chunk_len], f_agg)()\n        for i in range(int(np.ceil(len(x) / chunk_len)))\n    ]\n\ndef _into_subchunks(x, subchunk_length, every_n=1):\n    \"\"\"Split the time series x into subwindows of length \"subchunk_length\", starting every \"every_n\".\"\"\"\n    len_x = len(x)\n\n    assert subchunk_length > 1\n    assert every_n > 0\n\n    # how often can we shift a window of size subchunk_length over the input?\n    num_shifts = (len_x - subchunk_length) // every_n + 1\n    shift_starts = every_n * np.arange(num_shifts)\n    indices = np.arange(subchunk_length)\n\n    indexer = np.expand_dims(indices, axis=0) + np.expand_dims(shift_starts, axis=1)\n    return np.asarray(x)[indexer]\n\ndef set_property(key, value):\n    \"\"\"\n    This method returns a decorator that sets the property key of the function to value\n    \"\"\"\n\n    def decorate_func(func):\n        setattr(func, key, value)\n        if func.__doc__ and key == \"fctype\":\n            func.__doc__ = (\n                func.__doc__ + \"\\n\\n    *This function is of type: \" + value + \"*\\n\"\n            )\n        return func\n\n    return decorate_func\n\ndef log_return(list_stock_prices):\n    return np.log(list_stock_prices).diff()\n\ndef calc_wap(df):\n    wap = (df['bid_price1'] * df['ask_size1'] + df['ask_price1'] * df['bid_size1'])/(df['bid_size1'] + df['ask_size1'])\n    return wap\n\ndef mean(x):\n    return np.mean(x)\n\ndef length(x):\n    return len(x)\n\ndef standard_deviation(x):\n    return np.std(x)\n\ndef variation_coefficient(x):\n    mean = np.mean(x)\n    if mean != 0:\n        return np.std(x) / mean\n    else:\n        return np.nan\n    \ndef has_duplicate_max(x):\n    return np.sum(x == np.max(x)) >= 2\n\ndef has_duplicate_min(x):\n    return np.sum(x == np.min(x)) >= 2\n\ndef has_duplicate(x):\n    return x.size != np.unique(x).size\n\ndef log_return(list_stock_prices):\n    return np.log(list_stock_prices).diff() \n\ndef mean_n_absolute_max(x, number_of_maxima = 1):\n    \"\"\" Calculates the arithmetic mean of the n absolute maximum values of the time series.\"\"\"\n    assert (\n        number_of_maxima > 0\n    ), f\" number_of_maxima={number_of_maxima} which is not greater than 1\"\n\n    n_absolute_maximum_values = np.sort(np.absolute(x))[-number_of_maxima:]\n    return np.mean(n_absolute_maximum_values) if len(x) > number_of_maxima else np.NaN\n\ndef count_above(x, t):\n    if len(x)==0:\n        return np.nan\n    else:\n        return np.sum(x >= t) / len(x)\n\ndef count_below(x, t):\n    if len(x)==0:\n        return np.nan\n    else:\n        return np.sum(x <= t) / len(x)\n\n#number of valleys = number_peaks(-x, n)\ndef number_peaks(x, n):\n    \"\"\"\n    Calculates the number of peaks of at least support n in the time series x. A peak of support n is defined as a\n    subsequence of x where a value occurs, which is bigger than its n neighbours to the left and to the right.\n    \"\"\"\n    x_reduced = x[n:-n]\n\n    res = None\n    for i in range(1, n + 1):\n        result_first = x_reduced > _roll(x, i)[n:-n]\n\n        if res is None:\n            res = result_first\n        else:\n            res &= result_first\n\n        res &= x_reduced > _roll(x, -i)[n:-n]\n    return np.sum(res)\n\ndef longest_strike_below_mean(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    return np.max(_get_length_sequences_where(x < np.mean(x))) if x.size > 0 else 0\n\ndef longest_strike_above_mean(x):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    return np.max(_get_length_sequences_where(x > np.mean(x))) if x.size > 0 else 0\n\ndef quantile(x, q):\n    if len(x) == 0:\n        return np.NaN\n    return np.quantile(x, q)\n\n# crossing the mean ? other levels ? \ndef number_crossing_m(x, m):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    # From https://stackoverflow.com/questions/3843017/efficiently-detect-sign-changes-in-python\n    positive = x > m\n    return np.where(np.diff(positive))[0].size\n\ndef value_count(x, value):\n    if not isinstance(x, (np.ndarray, pd.Series)):\n        x = np.asarray(x)\n    if np.isnan(value):\n        return np.isnan(x).sum()\n    else:\n        return x[x == value].size\n\ndef range_count(x, min, max):\n    return np.sum((x >= min) & (x < max))","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.008994Z","iopub.execute_input":"2022-01-05T11:15:45.009319Z","iopub.status.idle":"2022-01-05T11:15:45.087442Z","shell.execute_reply.started":"2022-01-05T11:15:45.009275Z","shell.execute_reply":"2022-01-05T11:15:45.086649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Difine methods list\nbase_stats = [mean,sum,length,standard_deviation,variation_coefficient,variance,skewness,kurtosis]\nhigher_order_stats = [abs_energy,root_mean_square,sum_values,realized_volatility,realized_abs_skew,realized_skew,realized_vol_skew,realized_quarticity]\nmin_median_max = [minimum,median,maximum]\nadditional_quantiles = [quantile_01,quantile_025,quantile_075,quantile_09]\nother_min_max = [absolute_maximum,max_over_min,max_over_min_sq]\nmin_max_positions = [last_location_of_maximum,first_location_of_maximum,last_location_of_minimum,first_location_of_minimum]\npeaks = [number_peaks_2, mean_n_absolute_max_2, number_peaks_5, mean_n_absolute_max_5, number_peaks_10, mean_n_absolute_max_10]\ncounts = [count_unique,count,count_above_0,count_below_0,value_count_0,count_near_0]\nreoccuring_values = [count_above_mean,count_below_mean,percentage_of_reoccurring_values_to_all_values,percentage_of_reoccurring_datapoints_to_all_datapoints,sum_of_reoccurring_values,sum_of_reoccurring_data_points,ratio_value_number_to_time_series_length]\ncount_duplicates = [count_duplicate,count_duplicate_min,count_duplicate_max]\nvariations = [mean_abs_change,mean_change,mean_second_derivative_central,absolute_sum_of_changes,number_crossing_0]\nranges = [variance_std_ratio,ratio_beyond_01_sigma,ratio_beyond_02_sigma,ratio_beyond_03_sigma,large_standard_deviation,range_ratio]\nget_first_fn = [get_first,get_last]\ndraw_functions = [maximum_drawdown,maximum_drawup,drawdown_duration,drawup_duration]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.088538Z","iopub.execute_input":"2022-01-05T11:15:45.089373Z","iopub.status.idle":"2022-01-05T11:15:45.115039Z","shell.execute_reply.started":"2022-01-05T11:15:45.089329Z","shell.execute_reply":"2022-01-05T11:15:45.114241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_agg_functions_ever =\\\n    base_stats \\\n    + higher_order_stats \\\n    + min_median_max \\\n    + additional_quantiles \\\n    + other_min_max \\\n    + min_max_positions \\\n    + peaks + counts \\\n    + variations + ranges \\\n    + count_duplicates \\\n    + reoccuring_values \\\n    + get_first_fn \\\n    + draw_functions\nfor i in all_agg_functions_ever: print(i.__name__)","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.116115Z","iopub.execute_input":"2022-01-05T11:15:45.117403Z","iopub.status.idle":"2022-01-05T11:15:45.141898Z","shell.execute_reply.started":"2022-01-05T11:15:45.117364Z","shell.execute_reply":"2022-01-05T11:15:45.140652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO: Try different features here!\n# This is for you to play around, add or remove lists here to extract different featutres out of different sequences\nbaseline_feature_dict = {\n    'Open':           min_median_max,\n    'High':           [],\n    'Low':            [],\n    'Close':          [], #base_stats + higher_order_stats,\n    'Count':          [],\n    'Volume':         [],\n    'VWAP':           [],\n}\n\ncreate_feature_dict = baseline_feature_dict","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.145966Z","iopub.execute_input":"2022-01-05T11:15:45.146232Z","iopub.status.idle":"2022-01-05T11:15:45.151761Z","shell.execute_reply.started":"2022-01-05T11:15:45.146202Z","shell.execute_reply":"2022-01-05T11:15:45.150750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Two new features from the competition tutorial\ndef upper_shadow(df):\n    return df['High'] - np.maximum(df['Close'], df['Open'])\n\ndef lower_shadow(df):\n    return np.minimum(df['Close'], df['Open']) - df['Low']\n\ndef hlco_ratio(df):\n    return (df['High']-df['Low']) / (df['Close']-df['Open'])","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.153789Z","iopub.execute_input":"2022-01-05T11:15:45.154150Z","iopub.status.idle":"2022-01-05T11:15:45.168527Z","shell.execute_reply.started":"2022-01-05T11:15:45.154103Z","shell.execute_reply":"2022-01-05T11:15:45.167501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df):    \n    df_feat = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP', 'Asset_ID', 'date']].copy()\n    ## Adding some more features(2021/01/04)\n    df_feat['Upper_Shadow'] = upper_shadow(df_feat)\n    df_feat['Lower_Shadow'] = lower_shadow(df_feat)\n    # df_feat['hlco_ratio'] = hlco_ratio(df_feat)\n    \n    df_feat[\"Close/Open\"] = df_feat[\"Close\"] / df_feat[\"Open\"] \n    df_feat[\"Close-Open\"] = df_feat[\"Close\"] - df_feat[\"Open\"] \n    df_feat[\"High-Low\"] = df_feat[\"High\"] - df_feat[\"Low\"] \n    df_feat[\"High/Low\"] = df_feat[\"High\"] / df_feat[\"Low\"]\n    \n    df_feat['Mean'] = df_feat[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    df_feat[\"Median\"] = df_feat[[\"Open\", \"High\", \"Low\", \"Close\"]].median(axis=1)\n    ## (2021/01/04)\n    \n    try:\n        df_feat['window'] = df_feat['date'].dt.day.astype(str) # date列の日をwindowに入れる。\n    except:\n        df_feat['window'] = str(df_feat['date'].day)\n        \n    df_feat.drop(columns = 'date', inplace = True)        \n    grp = df_feat.groupby('window').agg(create_feature_dict) # FeatureEng \n    df_feat = df_feat.set_index('window').merge(grp, left_index=True, right_index=True).reset_index()\n    df_feat.columns = ['_'.join(col) if type(col) is not str else col for col in df_feat.columns]    \n    if 'window' in df_feat.columns: df_feat.drop(columns = 'window', inplace = True)    \n    return df_feat","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.170557Z","iopub.execute_input":"2022-01-05T11:15:45.170998Z","iopub.status.idle":"2022-01-05T11:15:45.181841Z","shell.execute_reply.started":"2022-01-05T11:15:45.170951Z","shell.execute_reply":"2022-01-05T11:15:45.181193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configure the model","metadata":{}},{"cell_type":"code","source":"# Numpy Version\ndef corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n\n# TF Version\ndef tf_cov(x, y, w): \n    return (tf.reduce_sum(w * (x - tf.reduce_mean(x * w)) * (y - tf.reduce_mean(y * w))) / tf.reduce_sum(w))\ndef tf_comp_metric(a, b, w): \n    return tf_cov(a, b, w) / tf.sqrt(tf_cov(a, a, w) * tf_cov(b, b, w))\ndef nn_comp_metric(w): \n    def wcorr(x, y): \n        return tf_comp_metric(x, y ,w)\n    return wcorr","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.182767Z","iopub.execute_input":"2022-01-05T11:15:45.184286Z","iopub.status.idle":"2022-01-05T11:15:45.203165Z","shell.execute_reply.started":"2022-01-05T11:15:45.184240Z","shell.execute_reply":"2022-01-05T11:15:45.202018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"### Time Series Cross Validation","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n            if self.verbose > 0:\n                    pass\n\n            yield [int(i) for i in train_array], [int(i) for i in test_array]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.204949Z","iopub.execute_input":"2022-01-05T11:15:45.205545Z","iopub.status.idle":"2022-01-05T11:15:45.224065Z","shell.execute_reply.started":"2022-01-05T11:15:45.205496Z","shell.execute_reply":"2022-01-05T11:15:45.222476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n            if self.verbose > 0:\n                    pass\n\n            yield [int(i) for i in train_array], [int(i) for i in test_array]","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.226208Z","iopub.execute_input":"2022-01-05T11:15:45.226743Z","iopub.status.idle":"2022-01-05T11:15:45.244555Z","shell.execute_reply.started":"2022-01-05T11:15:45.226695Z","shell.execute_reply":"2022-01-05T11:15:45.243584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main Training Function","metadata":{}},{"cell_type":"code","source":"def replace_outlier(df):\n    #四分位数\n    q1 = df.quantile(.01)\n    q3 = df.quantile(.99)\n    \n    #外れ値をクリップする\n    df = df.clip(q1, q3, axis=1)\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.245785Z","iopub.execute_input":"2022-01-05T11:15:45.246250Z","iopub.status.idle":"2022-01-05T11:15:45.262364Z","shell.execute_reply.started":"2022-01-05T11:15:45.246217Z","shell.execute_reply":"2022-01-05T11:15:45.261452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"######################\n# Parameters of LGBM\n######################\nbest_params = {\n    \"objective\": \"regression\",\n    \"n_estimators\" : 637,     # <-- (9) change from 200 to 500\n    \"boosting_type\" : \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\" : 800,       # <-- (10) Added parameter\n    \"learning_rate\" : 0.092534,   # <-- (10) Added parameter\n    \"colsample_bytree\":0.738963,\n    \"random_seed\" : 2022,\n    \"lambda_l1\" : 1,\n    \"lambda_l2\" : 1, \n    # \"tree_method\" : 'gpu_hist',  # THE MAGICAL PARAMETER\n    }","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.263355Z","iopub.execute_input":"2022-01-05T11:15:45.263568Z","iopub.status.idle":"2022-01-05T11:15:45.276988Z","shell.execute_reply.started":"2022-01-05T11:15:45.263538Z","shell.execute_reply":"2022-01-05T11:15:45.274767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import RobustScaler\n# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2\nglobal features\n\ndef get_Xy_and_model_for_asset(asset_id):\n    df = load_training_data_for_asset(asset_id)\n    df.reset_index(drop=True, inplace=True) # Fixed220104\n    df_proc = get_features(df)\n    df_proc['date'] = df['date'].copy()\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    groups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X = X.drop(columns = 'date')\n    # Clipping (add 220104)\n    X = replace_outlier(X)\n    # Scaler (add 220104)\n    scaler = RobustScaler().fit(X)\n    tmp_X = scaler.transform(X)\n    X = pd.DataFrame(tmp_X, columns=X.columns)\n    oof_preds = np.zeros(len(X))\n    \n    scores, models = [], []\n    for fold, (train_idx, val_idx) in enumerate(PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size = MAX_TRAIN_GROUP_SIZE, max_test_group_size = MAX_TEST_GROUP_SIZE).split(X, y, groups)):\n        # GET TRAINING, VALIDATION SET\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        global features\n        features = list(X.columns)\n        \n        # DISPLAY FOLD INFO\n        # if DEVICE == 'TPU':\n        #     if tpu: tf.tpu.experimental.initialize_tpu_system(tpu)\n        print('#'*25)\n        print('#### FOLD',fold+1)       \n        \n        # BUILD MODEL and TRAIN\n        model = LGBMRegressor(**best_params)       \n        model.fit(x_train, y_train)     \n\n        # PREDICT OOF\n        pred = model.predict(x_val)\n        models.append(model) \n        \n        # REPORT RESULTS\n        try: \n            mse = mean_squared_error(np.nan_to_num(y_val), np.nan_to_num(pred))\n        except: \n            mse = 0.0\n        scores.append(mse)\n        oof_preds[val_idx] = pred\n        w_score = corr(np.nan_to_num(y_val), \n                       np.nan_to_num(pred.flatten()), \n                       np.array([asset_weight_dict[asset_id]] * len(y_val))\n                       )\n        print('#### FOLD %i OOF MSE %.3f | WCORR: %.3f' % (fold + 1, mse, w_score))\n\n    orig_close = df['Close'].copy()\n    df = df_proc\n    df['Close'] = orig_close\n    df['oof_preds'] = np.nan_to_num(oof_preds)\n    print('\\n\\n' + ('-' * 80) + '\\n' + 'Finished training %s. ' % asset_name_dict[asset_id])\n    print('Results:')\n    print('Model: r2_score: %s | pearsonr: %s | wcorr: %s ' % (\n        r2_score(df['y'], df['oof_preds']), \n        pearsonr(df['y'], df['oof_preds'])[0], \n        corr(df['y'].values, df['oof_preds'].values, np.array([asset_weight_dict[asset_id]] * len(df['y'].values)))\n        ))\n    print('Predictions std: %s | Target std: %s' % (df['oof_preds'].std(), df['y'].std()))\n    \n    try: \n        plt.close()\n    except: \n        pass   \n    df2 = df.reset_index().set_index('date')\n    \n    fig = plt.figure(figsize = (12, 6))\n    # fig, ax_left = plt.subplots(figsize = (12, 6))\n    ax_left = fig.add_subplot(111)\n    ax_left.set_facecolor('azure')    \n    ax_right = ax_left.twinx()\n    # 3ヶ月窓ごとの相関係数(Targetと予測値の)->赤線\n    ax_left.plot(df2['y'].rolling(3 * 30 * 24 * 60).corr(df2['oof_preds']).iloc[::24 * 60], color = 'crimson', alpha=0.8, label = \"Target WCorr\")\n    # 単純なClose値\n    ax_right.plot(df2['Close'].iloc[::24 * 60], color = 'cornflowerblue', alpha=0.8, label = \"%s Close\" % asset_name_dict[asset_id])\n    h1, l1 = ax_left.get_legend_handles_labels()\n    h2, l2 = ax_right.get_legend_handles_labels()\n    ax_left.set_ylabel(r'Target WCorr')\n    ax_right.set_ylabel(r'Close')\n\n    plt.grid()\n    plt.legend(h1+h2, l1+l2, loc='lower right')\n    plt.xlabel('Time')\n    plt.title('3 month rolling pearsonr for %s' % (asset_name_dict[asset_id]))\n    plt.show()\n\n    fig = plt.figure(figsize = (12, 6))    \n    feature_importances = []\n    for model in (models):\n        feature_importances.append(model.feature_importances_)\n    feature_importances = np.array(feature_importances).mean(axis=0)\n\n    importances = pd.DataFrame({'Feature': X.columns,\n                                'Importance': np.round(feature_importances, 3)})\n\n    importances = importances.head(15).sort_values('Importance', ascending = True).set_index('Feature')\n\n    importances.plot.barh(color = 'deepskyblue', edgecolor = 'firebrick', legend = False)\n    plt.title('Feature importances for %s' % (asset_name_dict[asset_id]))\n    plt.xlabel('Importance');\n    plt.show()\n    \n    return scores, oof_preds, models, y, scaler","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.280092Z","iopub.execute_input":"2022-01-05T11:15:45.280497Z","iopub.status.idle":"2022-01-05T11:15:45.304777Z","shell.execute_reply.started":"2022-01-05T11:15:45.280437Z","shell.execute_reply":"2022-01-05T11:15:45.303910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training\nmodels, scores, targets, oof_preds, scalers = {}, {}, {}, {}, {}\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    try:\n        print(f\"Training model for {asset_name:<16} (ID={asset_id:<2})\")\n        cur_scores, cur_oof_preds, cur_models, cur_targets, scaler = get_Xy_and_model_for_asset(asset_id)\n        scores[asset_id], oof_preds[asset_id], models[asset_id], targets[asset_id], scalers[asset_id] = np.mean(cur_scores), cur_oof_preds, cur_models, cur_targets, scaler\n    except: pass","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:15:45.306051Z","iopub.execute_input":"2022-01-05T11:15:45.306371Z","iopub.status.idle":"2022-01-05T11:28:58.529598Z","shell.execute_reply.started":"2022-01-05T11:15:45.306331Z","shell.execute_reply":"2022-01-05T11:28:58.528361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate OOF MSE","metadata":{}},{"cell_type":"code","source":"# COMPUTE OVERALL OOF MSE\nprint('Overall MEAN OOF MSE %s' % np.mean(list(scores.values())))\n\n# SAVE OOF TO DISK \ny_pred, y_true, weights = [], [], []\nfor asset in oof_preds:\n    df_oof = pd.DataFrame(dict(asset_id = asset, oof_preds=oof_preds[asset]))\n    df_oof.to_csv(str(asset) + '_oof.csv',index=False)\n    y_pred += oof_preds[asset].tolist()\n    y_true += targets[asset].tolist() \n    weights += ([asset_weight_dict[asset]] * len(oof_preds[asset].tolist()))\n    print('%s score: %s' % (asset_name_dict[asset], corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten()))))\n    \nprint('Overall score %s' % corr(np.nan_to_num(np.array(y_true).flatten()), np.nan_to_num(np.array(y_pred).flatten()), np.nan_to_num(np.array(weights).flatten())))","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:28:58.531069Z","iopub.execute_input":"2022-01-05T11:28:58.531952Z","iopub.status.idle":"2022-01-05T11:30:39.015642Z","shell.execute_reply.started":"2022-01-05T11:28:58.531885Z","shell.execute_reply":"2022-01-05T11:30:39.014602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <span class=\"title-section w3-xxlarge\" id=\"submit\">Submit To Kaggle 🇰</span><hr>","metadata":{}},{"cell_type":"code","source":"all_df_test = []\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    # Feature Eng.\n    df_test['date'] = pd.to_datetime(df_test['timestamp'], unit = 's')\n    df_proc = get_features(df_test)\n    df_proc['date'] = df_test['date'].copy()\n    # df_proc['y'] = df['Target']\n    # df_proc = df_proc.dropna(how=\"any\")\n    # X = df_proc.drop(\"y\", axis=1)\n    # y = df_proc[\"y\"]\n    groups = pd.factorize(df_proc['date'].dt.day.astype(str) + '_' + df_proc['date'].dt.month.astype(str) + '_' + df_proc['date'].dt.year.astype(str))[0]\n    df_proc = df_proc.drop(columns = 'date')\n    \n    # df_procを1行ずつ処理\n    # j-idx, row-data(1行分)\n    for j , row in df_proc.iterrows():\n        model = models[row['Asset_ID']]   # Asset_IDに対応するモデルLISTを呼び出す\n        scaler = scalers[row['Asset_ID']] # Asset_IDに対応するSCALERを呼び出す\n        # x_test = get_features(row)      # 説明変数を呼び込む\n        x_test = row.to_frame().T \n        tmp_X = scaler.transform(x_test)\n        x_test = pd.DataFrame(tmp_X, columns=x_test.columns)\n        # inference\n        y_preds = []\n        for fold in range(FOLDS):\n            y_pred = model[fold].predict(x_test)\n            y_preds.append(y_pred)\n        y_pred = np.array(y_preds).mean()\n\n        row['row_id'] = int(df_pred.row_id.iloc[j])\n        df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n\n        # Print just one sample row to get a feeling of what it looks like\n        if i == 0 and j == 0:\n            display(x_test)\n\n    # Display the first prediction dataframe\n    if i == 0:\n        display(df_pred)\n    all_df_test.append(df_test)\n\n    # Send submissions/ ここでpredすることで、次のイテレーションに進める。\n    env.predict(df_pred) ","metadata":{"execution":{"iopub.status.busy":"2022-01-05T11:30:39.017299Z","iopub.execute_input":"2022-01-05T11:30:39.017594Z","iopub.status.idle":"2022-01-05T11:30:40.411766Z","shell.execute_reply.started":"2022-01-05T11:30:39.017557Z","shell.execute_reply":"2022-01-05T11:30:40.411058Z"},"trusted":true},"execution_count":null,"outputs":[]}]}