{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# INPUT","metadata":{}},{"cell_type":"code","source":"import os, gc, warnings, random, datetime, traceback, gresearch_crypto\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd, numpy as np\n\nimport numpy.polynomial.hermite as Herm\nimport math\n\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom matplotlib.colors import ListedColormap\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nfrom sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:33:00.673896Z","iopub.execute_input":"2022-02-13T17:33:00.674333Z","iopub.status.idle":"2022-02-13T17:33:07.28198Z","shell.execute_reply.started":"2022-02-13T17:33:00.674234Z","shell.execute_reply":"2022-02-13T17:33:07.281259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV         = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\nEXAMPLE_TEST      = '/kaggle/input/g-research-crypto-forecasting/example_test.csv'\n\ndf_train          = pd.read_csv(TRAIN_CSV)\ndf_test           = pd.read_csv(EXAMPLE_TEST)\ndf_asset_details  = pd.read_csv(ASSET_DETAILS_CSV).sort_values('Asset_ID')\n\nurl_save_weight   = './'\nurl_load_weight   = '../input/3f-weight-mlp512'","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:33:07.283861Z","iopub.execute_input":"2022-02-13T17:33:07.284402Z","iopub.status.idle":"2022-02-13T17:34:17.664779Z","shell.execute_reply.started":"2022-02-13T17:33:07.284353Z","shell.execute_reply":"2022-02-13T17:34:17.663255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:17.667388Z","iopub.execute_input":"2022-02-13T17:34:17.667668Z","iopub.status.idle":"2022-02-13T17:34:17.67476Z","shell.execute_reply.started":"2022-02-13T17:34:17.667629Z","shell.execute_reply":"2022-02-13T17:34:17.67364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"TPU\" \n#device = \"GPU\" \n\nseed = 42\nfix_all_seeds(seed)\n\nremove_op_test_overlapping_data = True #& False \nvisualization                   = True #& False\ntrain_models                    = True #& False   #False == Load Models    \ntest_baseline_model             = True #& False\nfine_tuning                     = True #& False  #&True == train_models  #pre training 10 epochs \ntuning_id                       = [3,12,13]      #fine tuning asset id 3,12,13","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:17.676476Z","iopub.execute_input":"2022-02-13T17:34:17.676883Z","iopub.status.idle":"2022-02-13T17:34:17.688779Z","shell.execute_reply.started":"2022-02-13T17:34:17.676837Z","shell.execute_reply":"2022-02-13T17:34:17.687963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if device == \"TPU\":\n    print(\"connecting to TPU...\")\n    try:\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n    if tpu:\n        try:\n            print(\"initializing  TPU ...\")\n            tf.config.experimental_connect_to_cluster(tpu)\n            tf.tpu.experimental.initialize_tpu_system(tpu)\n            strategy = tf.distribute.TPUStrategy(tpu)\n            print(\"TPU initialized\")\n        except: print(\"failed to initialize TPU\")\n    else: device = \"GPU\"\n\nif device != \"TPU\": strategy = tf.distribute.get_strategy()\nif device == \"GPU\": print(\"Num GPUs Available: \", len(tf.config.experimental.list_physical_devices('GPU')))\nAUTO     = tf.data.experimental.AUTOTUNE\nREPLICAS = strategy.num_replicas_in_sync","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:17.69168Z","iopub.execute_input":"2022-02-13T17:34:17.692287Z","iopub.status.idle":"2022-02-13T17:34:23.686369Z","shell.execute_reply.started":"2022-02-13T17:34:17.692231Z","shell.execute_reply":"2022-02-13T17:34:23.685427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env       = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:23.687712Z","iopub.execute_input":"2022-02-13T17:34:23.68831Z","iopub.status.idle":"2022-02-13T17:34:23.693747Z","shell.execute_reply.started":"2022-02-13T17:34:23.688265Z","shell.execute_reply":"2022-02-13T17:34:23.692822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CV PARAMS\nFOLDS                = 3\nGROUP_GAP            = 180\nMAX_TEST_GROUP_SIZE  = 270\nMAX_TRAIN_GROUP_SIZE = 450\n\n# BATCH SIZE AND EPOCHS\nBATCH_SIZES = [2**9] * FOLDS  \nEPOCHS      = [3] * FOLDS\n\n# WHICH NETWORK ARCHITECTURE TO USE?\nUNITS_NETWORK     = [2**7] * FOLDS\nUNITS_NETWORK_MLP = [2**6, 2**7, 2**8]  \n\n# USE VERBOSE=0 for silent, VERBOSE=1 for interactive, VERBOSE=2 for commit\nVERBOSE = 2","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:23.695044Z","iopub.execute_input":"2022-02-13T17:34:23.69531Z","iopub.status.idle":"2022-02-13T17:34:23.705733Z","shell.execute_reply.started":"2022-02-13T17:34:23.695281Z","shell.execute_reply":"2022-02-13T17:34:23.704984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove the features to test the LR baseline score.\nif remove_op_test_overlapping_data:\n    df_train['datetime'] = pd.to_datetime(df_train['timestamp'], unit='s')\n    df_valid  = df_train[(df_train['datetime'] > '2021-09-20 00:00:00')].reset_index(drop=True)\n    df_train  = df_train.drop(['datetime'],axis=1)\n    df_valid  = df_valid.drop(['datetime'],axis=1)\n    \nelse:\n    df_train['datetime'] = pd.to_datetime(df_train['timestamp'], unit='s')\n    df_valid  = df_train[(df_train['datetime'] > '2021-02-23 00:00:00')].reset_index(drop=True)\n    df_train  = df_train.drop(['datetime'],axis=1)\n    df_valid  = df_valid.drop(['datetime'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:23.707157Z","iopub.execute_input":"2022-02-13T17:34:23.707384Z","iopub.status.idle":"2022-02-13T17:34:26.468796Z","shell.execute_reply.started":"2022-02-13T17:34:23.707357Z","shell.execute_reply":"2022-02-13T17:34:26.467701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_valid = df_valid.dropna(subset=['Target']).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.469993Z","iopub.execute_input":"2022-02-13T17:34:26.470309Z","iopub.status.idle":"2022-02-13T17:34:26.482481Z","shell.execute_reply.started":"2022-02-13T17:34:26.470272Z","shell.execute_reply":"2022-02-13T17:34:26.481113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#@title GroupTimeSeriesSplit { display-mode: \"form\" }\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass GroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_size : int, default=None\n        Maximum size for a single training set.\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from sklearn.model_selection import GroupTimeSeriesSplit\n    >>> groups = np.array(['a', 'a', 'a', 'a', 'a', 'a',\\\n                           'b', 'b', 'b', 'b', 'b',\\\n                           'c', 'c', 'c', 'c',\\\n                           'd', 'd', 'd'])\n    >>> gtss = GroupTimeSeriesSplit(n_splits=3)\n    >>> for train_idx, test_idx in gtss.split(groups, groups=groups):\n    ...     print(\"TRAIN:\", train_idx, \"TEST:\", test_idx)\n    ...     print(\"TRAIN GROUP:\", groups[train_idx],\\\n                  \"TEST GROUP:\", groups[test_idx])\n    TRAIN: [0, 1, 2, 3, 4, 5] TEST: [6, 7, 8, 9, 10]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a']\\\n    TEST GROUP: ['b' 'b' 'b' 'b' 'b']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] TEST: [11, 12, 13, 14]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b']\\\n    TEST GROUP: ['c' 'c' 'c' 'c']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\\\n    TEST: [15, 16, 17]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b' 'c' 'c' 'c' 'c']\\\n    TEST GROUP: ['d' 'd' 'd']\n    \"\"\"\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_size=None\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_size = max_train_size\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n        group_test_size = n_groups // n_folds\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n            for train_group_idx in unique_groups[:group_test_start]:\n                train_array_tmp = group_dict[train_group_idx]\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n            train_end = train_array.size\n            if self.max_train_size and self.max_train_size < train_end:\n                train_array = train_array[train_end -\n                                          self.max_train_size:train_end]\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n            yield [int(i) for i in train_array], [int(i) for i in test_array]\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Allows for a gap in groups to avoid potentially leaking info from\n    train into test if the model has windowed or lag features.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n\n\n            if self.verbose > 0:\n                    pass\n\n            yield [int(i) for i in train_array], [int(i) for i in test_array]","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-02-13T17:34:26.484246Z","iopub.execute_input":"2022-02-13T17:34:26.484654Z","iopub.status.idle":"2022-02-13T17:34:26.525384Z","shell.execute_reply.started":"2022-02-13T17:34:26.484613Z","shell.execute_reply":"2022-02-13T17:34:26.524429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#simple units\nm=1.0\nw=1.0\nhbar=1.0\n\ndef hermite(x, n):\n    xi = np.sqrt(m*w/hbar)*x\n    herm_coeffs = np.zeros(n+1)\n    herm_coeffs[n] = 1\n    return Herm.hermval(xi, herm_coeffs)\n\ndef stationary_state(x,n):\n    xi = np.sqrt(m*w/hbar)*x\n    prefactor = 1.0/math.sqrt(2.0**n * math.factorial(n)) * (m*w/(np.pi*hbar))**(0.25)\n    psi = prefactor * np.exp(- xi**2 / 2) * hermite(x,n)\n    return psi","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.526464Z","iopub.execute_input":"2022-02-13T17:34:26.526716Z","iopub.status.idle":"2022-02-13T17:34:26.54218Z","shell.execute_reply.started":"2022-02-13T17:34:26.526688Z","shell.execute_reply":"2022-02-13T17:34:26.541128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, submissions=False):\n    dff = df[['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']].copy()\n    \n    dff['upper_Shadow'] = dff['High'] - np.maximum(dff['Close'], dff['Open'])  \n    dff['lower_Shadow'] = np.minimum(dff['Close'], dff['Open']) - dff['Low']                  \n    dff['hlco_ration']  = (dff['High'] - dff['Low'])/(dff['Close']-dff['Open'])\n    dff['high_div_low'] = dff['High'] / dff['Low']\n    dff['gtrade']       = (dff['Close'] - dff['Open']) / dff['Count']\n    dff['shadow1']      = (dff['Close'] - dff['Open']) / dff['Volume']\n    dff['shadow3']      = dff['upper_Shadow'] / dff['Volume']\n    dff['shadow5']      = dff['lower_Shadow'] / dff['Volume']\n    dff['mean2']        = (dff['shadow1'] + dff['Volume']) / 2\n    dff['spread']       = dff['High'] - dff['Low']\n    dff['log_exp_co']   = np.logaddexp(dff['Close'], dff['Open'])\n    dff['volume_count'] = dff['Volume'] / (dff['Count'] + 1) \n\n    #Autoencoder(AE) with an applied quantum harmonic oscillator(QHO) for indicator\n    dff['harmonic_oscillator_115v'] = stationary_state(dff['volume_count'], 115) \n    #114.59155903 == 360/pi, 6h*60min/pi ; 6h = 1/4 of day ≈ 115+-5 so 110 <= n >= 120 for training \n    #Bias in dataset Harmonic-Oscillator If the value is prohibitively exorbitant, you can adjust the hermite value.\n    #Add :: df['hermite_n'] = hermite(df, n) ;In simple units, recommend 60 <= n >= 210\n    #There might be a better value, or other factors could be used to adjust for the value and produce better outcomes.\n     \n    #drop & replace\n    if submissions:\n        dff = dff.drop(['VWAP', 'Close', 'Low', 'High', 'Open', 'Volume', 'Count']) \n    else:\n        dff = dff.drop(['VWAP', 'Close', 'Low', 'High', 'Open', 'Volume', 'Count'],axis=1) \n    dff = dff.replace([np.inf, -np.inf, np.nan], 0)\n    return dff","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.543967Z","iopub.execute_input":"2022-02-13T17:34:26.545386Z","iopub.status.idle":"2022-02-13T17:34:26.558731Z","shell.execute_reply.started":"2022-02-13T17:34:26.545342Z","shell.execute_reply":"2022-02-13T17:34:26.557727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_cv_indices(cv, X, y, group, ax, n_splits, lw=10):\n    cmap_cv = plt.cm.coolwarm\n    jet     = plt.cm.get_cmap('jet', 256)\n    seq     = np.linspace(0, 1, 256)\n    _       = np.random.shuffle(seq)   # inplace\n    cmap_data = ListedColormap(jet(seq))    \n    for ii, (tr, tt) in enumerate(list(cv.split(X=X, y=y, groups=group))):\n        indices = np.array([np.nan] * len(X))\n        indices[tt] = 1\n        indices[tr] = 0        \n        ax.scatter(range(len(indices)), [ii + .5] * len(indices), c=indices, marker='_', lw=lw, cmap=cmap_cv, vmin=-.2, vmax=1.2)\n    ax.scatter(range(len(X)), [ii + 1.5] * len(X), c=y, marker='_', lw=lw, cmap=plt.cm.Set3)\n    ax.scatter(range(len(X)), [ii + 2.5] * len(X), c=group, marker='_', lw=lw, cmap=cmap_data)\n    yticklabels = list(range(n_splits)) + ['target', 'day']\n    ax.set(yticks=np.arange(n_splits+2) + .5, yticklabels=yticklabels, xlabel='Sample index', ylabel=\"CV iteration\", ylim=[n_splits+2.2, -.2], xlim=[0, len(y)])\n    ax.set_title('{}'.format(type(cv).__name__), fontsize=15)\n    return ax","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.559943Z","iopub.execute_input":"2022-02-13T17:34:26.560275Z","iopub.status.idle":"2022-02-13T17:34:26.577805Z","shell.execute_reply.started":"2022-02-13T17:34:26.560241Z","shell.execute_reply":"2022-02-13T17:34:26.576969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import backend as K\nclass Mish(tf.keras.layers.Layer):\n\n    def __init__(self, **kwargs):\n        super(Mish, self).__init__(**kwargs)\n        self.supports_masking = True\n\n    def call(self, inputs):\n        return inputs * K.tanh(K.softplus(inputs))\n\n    def get_config(self):\n        base_config = super(Mish, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\ndef mish(x):\n\treturn tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)))(x)\n\ntf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.581501Z","iopub.execute_input":"2022-02-13T17:34:26.581805Z","iopub.status.idle":"2022-02-13T17:34:26.603163Z","shell.execute_reply.started":"2022-02-13T17:34:26.581771Z","shell.execute_reply":"2022-02-13T17:34:26.602285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(fold, dim = 128, weight = 1.0):\n\n    hidden_units      = UNITS_NETWORK[fold]\n    dropout_rates     = 0.25\n    dropout_rates_mlp = 0.4\n    lr                = 9e-4\n\n    inp = tf.keras.layers.Input(shape = (dim, ))\n    x0  = tf.keras.layers.BatchNormalization()(inp)\n\n    encoder = tf.keras.layers.GaussianNoise(dropout_rates)(x0)\n    encoder = tf.keras.layers.Dense(hidden_units)(encoder)\n    encoder = tf.keras.layers.BatchNormalization()(encoder)\n    encoder = tf.keras.layers.Activation('mish')(encoder)\n\n    decoder = tf.keras.layers.Dropout(dropout_rates)(encoder)\n    decoder = tf.keras.layers.Dense(dim, name = 'decoder')(decoder)\n\n    x_ae = tf.keras.layers.Dense(hidden_units)(decoder)\n    x_ae = tf.keras.layers.BatchNormalization()(x_ae)\n    x_ae = tf.keras.layers.Activation('mish')(x_ae)\n    x_ae = tf.keras.layers.Dropout(dropout_rates)(x_ae)\n\n    out_ae = tf.keras.layers.Dense(1, activation = 'mish', name = 'ae_out')(x_ae)\n\n    x = tf.keras.layers.Concatenate()([x0, encoder])\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dropout(dropout_rates)(x)\n  \n    for i in range(len(UNITS_NETWORK_MLP)):\n        x = tf.keras.layers.Dense(UNITS_NETWORK_MLP[i])(x)\n        x = tf.keras.layers.BatchNormalization()(x)\n        x = tf.keras.layers.Activation('mish')(x)\n        x = tf.keras.layers.Dropout(dropout_rates)(x)\n\n    out = tf.keras.layers.Dense(1, activation = 'mish', name = 'out')(x)\n\n    model = tf.keras.models.Model(inputs = inp, outputs = [decoder, out_ae, out])\n    model.compile(optimizer = tf.keras.optimizers.RMSprop(learning_rate = lr, momentum=0.09),\n                  loss = {'decoder': tf.keras.losses.CosineSimilarity(),\n                          'ae_out':  tf.keras.losses.MeanSquaredError(),\n                          'out': mse_w(weight), \n                         },\n                  metrics = {'decoder': tf.keras.metrics.CosineSimilarity(name='cosine'),\n                             'ae_out':  tf.keras.metrics.RootMeanSquaredError(name='rmse'),\n                             'out':  tf.keras.metrics.RootMeanSquaredError(name='rmse'),\n                            },\n                 )    \n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.604701Z","iopub.execute_input":"2022-02-13T17:34:26.604924Z","iopub.status.idle":"2022-02-13T17:34:26.622622Z","shell.execute_reply.started":"2022-02-13T17:34:26.604898Z","shell.execute_reply":"2022-02-13T17:34:26.621528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mse(x, y): return  tf.keras.losses.mean_squared_error(x, y)\n\ndef weight_mse(x, y, w): \n    sum_ = 0\n    sum_ += math.sqrt(w)*mse(x, y)\n    return tf.reduce_sum(sum_) \n\ndef mse_w(w): \n    def w_mse(x, y): return weight_mse(x, y ,w)\n    return w_mse","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.624298Z","iopub.execute_input":"2022-02-13T17:34:26.624563Z","iopub.status.idle":"2022-02-13T17:34:26.635029Z","shell.execute_reply.started":"2022-02-13T17:34:26.624532Z","shell.execute_reply":"2022-02-13T17:34:26.634135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_lr_callback(batch_size = 8):\n    lr_start   = 5e-6\n    lr_max     = 1.25e-6 * REPLICAS * batch_size\n    lr_min     = 1e-6\n    lr_ramp_ep = 5\n    lr_sus_ep  = 0\n    lr_decay   = 0.8\n    def lrfn(epoch):\n        if epoch < lr_ramp_ep: lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start\n        elif epoch < lr_ramp_ep + lr_sus_ep: lr = lr_max\n        else: lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min\n        return lr\n    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=False)\n    return lr_callback","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:26.636402Z","iopub.execute_input":"2022-02-13T17:34:26.636658Z","iopub.status.idle":"2022-02-13T17:34:26.651839Z","shell.execute_reply.started":"2022-02-13T17:34:26.636628Z","shell.execute_reply":"2022-02-13T17:34:26.650983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_all_in_model_for_asset(asset_id, df_train, df_asset_details, fine_tuning=False, tuning_id=tuning_id):\n\n    #weight\n    asset_weight    = {df_asset_details['Asset_ID'].tolist()[idx]: df_asset_details['Weight'].tolist()[idx] for idx in range(len(df_asset_details))}\n\n    df              = df_train[df_train['Asset_ID'] == asset_id]\n    df_proc         = get_features(df)\n    df_proc['date'] = pd.to_datetime(df['timestamp'], unit='s')\n    df_proc['y']    = df['Target']#.fillna(0)\n    df_proc         = df_proc.dropna(how=\"any\")\n    X               = df_proc.drop(\"y\", axis=1)\n    y               = df_proc[\"y\"]\n    groups          = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X               = X.drop(columns = 'date')\n    del df\n    del df_proc\n    rubbish = gc.collect()\n    gkf     = PurgedGroupTimeSeriesSplit(n_splits             = FOLDS,\n                                         group_gap            = GROUP_GAP, \n                                         max_train_group_size = MAX_TRAIN_GROUP_SIZE, \n                                         max_test_group_size  = MAX_TEST_GROUP_SIZE).split(X, y, groups)\n    \n    models = []\n    for fold, (train_idx, val_idx) in enumerate(gkf):\n        # GET TRAINING, VALIDATION SET\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        print('>>> FOLD:',fold); print('>>> Training: UNITS_NETWORK=%s | BATCH_SIZES=%s' % (UNITS_NETWORK[fold], BATCH_SIZES[fold]*REPLICAS))\n        \n        # BUILD MODEL\n        K.clear_session()\n        with strategy.scope(): model = build_model(fold, dim = x_train.shape[1], weight = asset_weight[asset_id])\n\n        # SAVE BEST MODEL EACH FOLD\n        model_save = tf.keras.callbacks.ModelCheckpoint(url_save_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold), \n                                                         monitor           = 'val_out_rmse',\n                                                         verbose           = 0, \n                                                         save_best_only    = True,\n                                                         save_weights_only = True,\n                                                         mode              = 'min',\n                                                         save_freq         = 'epoch')\n        early_stop = tf.keras.callbacks.EarlyStopping(monitor              = 'val_out_rmse', \n                                                      patience             = 1, \n                                                      mode                 = 'min', \n                                                      restore_best_weights = True)\n        # TRAIN\n        if not fine_tuning: \n            if fold == 0:\n                history = model.fit( x_train, y_train, \n                                    epochs          = EPOCHS[fold], \n                                    callbacks       = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                    validation_data = (x_val, y_val), \n                                    verbose         = VERBOSE)\n            elif fold == (FOLDS-1):\n                \n                model.load_weights(url_save_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold-1)) \n                history = model.fit( x_train, y_train, \n                                   epochs          = EPOCHS[fold], \n                                   callbacks       = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                   validation_data = (x_val, y_val), \n                                   verbose         = VERBOSE)\n                models.append(model)\n            else:\n                \n                model.load_weights(url_save_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold-1)) \n                history = model.fit( x_train, y_train, \n                                    epochs          = EPOCHS[fold], \n                                    callbacks       = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                    validation_data = (x_val, y_val), \n                                    verbose         = VERBOSE)\n        else:\n            model.load_weights(url_load_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold)) \n            model.trainable = False\n            for i in range(len(tuning_id)):\n                if asset_id == tuning_id[i]:\n                    model.trainable = True\n                    history = model.fit(x_train, y_train, \n                                        epochs         = EPOCHS[fold], \n                                        callbacks      = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                        validation_data= (x_val, y_val), \n                                        verbose        = VERBOSE) \n        print('>>> Loading: FOLD=%s' %(fold))\n        models.append(model)\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:35:39.870052Z","iopub.execute_input":"2022-02-13T17:35:39.870465Z","iopub.status.idle":"2022-02-13T17:35:39.897821Z","shell.execute_reply.started":"2022-02-13T17:35:39.870428Z","shell.execute_reply":"2022-02-13T17:35:39.896832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_Xy_and_model_for_asset(asset_id, df_train, df_asset_details, fine_tuning=False, tuning_id=tuning_id):\n\n    #weight\n    asset_weight    = {df_asset_details['Asset_ID'].tolist()[idx]: df_asset_details['Weight'].tolist()[idx] for idx in range(len(df_asset_details))}\n\n    df              = df_train[df_train['Asset_ID'] == asset_id]\n    df_proc         = get_features(df)\n    df_proc['date'] = pd.to_datetime(df['timestamp'], unit='s')\n    df_proc['y']    = df['Target']\n    df_proc         = df_proc.dropna(how=\"any\")\n    X               = df_proc.drop(\"y\", axis=1)\n    y               = df_proc[\"y\"]\n    groups          = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X               = X.drop(columns = 'date')\n    del df\n    del df_proc\n    \n    rubbish = gc.collect()\n    gkf     = PurgedGroupTimeSeriesSplit(n_splits             = FOLDS,\n                                         group_gap            = GROUP_GAP, \n                                         max_train_group_size = MAX_TRAIN_GROUP_SIZE, \n                                         max_test_group_size  = MAX_TEST_GROUP_SIZE).split(X, y, groups)\n    \n    models = []\n    for fold, (train_idx, val_idx) in enumerate(gkf):\n        # GET TRAINING, VALIDATION SET\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n\n        print('>>> FOLD:',fold); print('>>> Training: UNITS_NETWORK=%s | BATCH_SIZES=%s' % (UNITS_NETWORK[fold], BATCH_SIZES[fold]*REPLICAS))\n        \n        # BUILD MODEL\n        K.clear_session()\n        with strategy.scope(): model = build_model(fold, dim = x_train.shape[1], weight = asset_weight[asset_id])\n\n        # SAVE BEST MODEL EACH FOLD\n        model_save = tf.keras.callbacks.ModelCheckpoint(url_save_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold), \n                                                         monitor = 'val_out_rmse', verbose = 0, \n                                                         save_best_only = True, save_weights_only = True,\n                                                         mode = 'min', save_freq = 'epoch')\n        early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_out_rmse', patience=1, mode='min', restore_best_weights=True)\n        # TRAIN\n        if not fine_tuning:\n            history = model.fit( x_train, y_train, \n                                epochs          = EPOCHS[fold], \n                                callbacks       = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                validation_data = (x_val, y_val), \n                                verbose         = VERBOSE) \n        else:\n            model.load_weights(url_load_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold)) \n            model.trainable = False\n            for i in range(len(tuning_id)):\n                if asset_id == tuning_id[i]:\n                    model.trainable = True\n                    history = model.fit(x_train, y_train, \n                                        epochs         = EPOCHS[fold], \n                                        callbacks      = [model_save, get_lr_callback(BATCH_SIZES[fold]), early_stop], \n                                        validation_data= (x_val, y_val), \n                                        verbose        = VERBOSE) \n        print('>>> Loading: FOLD=%s' %(fold))\n        models.append(model)\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:35:41.849496Z","iopub.execute_input":"2022-02-13T17:35:41.849985Z","iopub.status.idle":"2022-02-13T17:35:41.870706Z","shell.execute_reply.started":"2022-02-13T17:35:41.849947Z","shell.execute_reply":"2022-02-13T17:35:41.869641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_load_model_for_asset(asset_id, df_train):\n\n    df              = df_train[df_train['Asset_ID'] == asset_id]\n    df_proc         = get_features(df)\n    df_proc['date'] = pd.to_datetime(df['timestamp'], unit='s')\n    df_proc['y']    = df['Target']\n    df_proc         = df_proc.dropna(how=\"any\")\n    X               = df_proc.drop(\"y\", axis=1)\n    y               = df_proc[\"y\"]\n    groups          = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X               = X.drop(columns = 'date')\n    rubbish         = gc.collect()\n    gkf             = PurgedGroupTimeSeriesSplit(n_splits             = FOLDS, \n                                                 group_gap            = GROUP_GAP, \n                                                 max_train_group_size = MAX_TRAIN_GROUP_SIZE, \n                                                 max_test_group_size  = MAX_TEST_GROUP_SIZE).split(X, y, groups)\n    models = []\n    for fold, (train_idx, val_idx) in enumerate(gkf):\n        x_train, x_val = X.iloc[train_idx], X.iloc[val_idx]\n        y_train, y_val = y.iloc[train_idx], y.iloc[val_idx]\n        \n        K.clear_session()\n        with strategy.scope(): model = build_model(fold, dim = x_train.shape[1])\n        model.load_weights(url_load_weight + '/id_%ifold-%i.hdf5' %(asset_id, fold)) \n        print('>>> Loading: FOLD=%s' %(fold))\n        models.append(model)\n    return models","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:35:52.292696Z","iopub.execute_input":"2022-02-13T17:35:52.293706Z","iopub.status.idle":"2022-02-13T17:35:52.304928Z","shell.execute_reply.started":"2022-02-13T17:35:52.293654Z","shell.execute_reply":"2022-02-13T17:35:52.304212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PROCESSING","metadata":{}},{"cell_type":"code","source":"if visualization:\n    asset_id = 1 # 1 == btc\n    df = df_train[df_train['Asset_ID'] == asset_id]\n    df_proc = get_features(df)\n    df_proc['date'] = pd.to_datetime(df['timestamp'], unit='s')\n    df_proc['y'] = df['Target']\n    df_proc = df_proc.dropna(how=\"any\")\n    X = df_proc.drop(\"y\", axis=1)\n    y = df_proc[\"y\"]\n    groups = pd.factorize(X['date'].dt.day.astype(str) + '_' + X['date'].dt.month.astype(str) + '_' + X['date'].dt.year.astype(str))[0]\n    X = X.drop(columns = 'date')\n\n    fig, ax = plt.subplots(figsize = (12, 6))\n    cv = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap = GROUP_GAP, max_train_group_size=MAX_TRAIN_GROUP_SIZE, max_test_group_size=MAX_TEST_GROUP_SIZE)\n    plot_cv_indices(cv, X, y, groups, ax, FOLDS, lw=20)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:36:00.508854Z","iopub.execute_input":"2022-02-13T17:36:00.509225Z","iopub.status.idle":"2022-02-13T17:38:36.680877Z","shell.execute_reply.started":"2022-02-13T17:36:00.509183Z","shell.execute_reply":"2022-02-13T17:38:36.679919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodels =  {}\nfor asset_id, asset_name in zip(df_asset_details['Asset_ID'], df_asset_details['Asset_Name']):\n    if train_models:\n        if not fine_tuning:\n            print('.'*128); print(f\"Training Model for {asset_name:<16} (ID={asset_id:<2})\")\n            curl_models = get_Xy_and_model_for_asset(asset_id, df_train, df_asset_details, fine_tuning=fine_tuning, tuning_id=tuning_id)\n            #curl_models = get_all_in_model_for_assetasset_id, df_train, df_asset_details, fine_tuning=fine_tuning, tuning_id=tuning_id)\n            models[asset_id] = curl_models\n        else:\n            print('.'*128); print(f\"Fine Tuning Model for {asset_name:<16} (ID={asset_id:<2})\")\n            curl_models = get_Xy_and_model_for_asset(asset_id, df_train, df_asset_details, fine_tuning=fine_tuning, tuning_id=tuning_id)\n            #curl_models = get_all_in_model_for_assetasset_id, df_train, df_asset_details, fine_tuning=fine_tuning, tuning_id=tuning_id)\n            models[asset_id] = curl_models     \n    else:\n        print('.'*42); print(f\"Loading of Model {asset_name:<16} (ID={asset_id:<2})\")\n        curl_models = get_load_model_for_asset(asset_id, df_train)\n        models[asset_id] = curl_models","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-13T17:34:27.074068Z","iopub.status.idle":"2022-02-13T17:34:27.07467Z","shell.execute_reply.started":"2022-02-13T17:34:27.074466Z","shell.execute_reply":"2022-02-13T17:34:27.074489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OUTPUT","metadata":{}},{"cell_type":"code","source":"id_test = [0,1,2,3,4,5,6,7,8,9,10,11,12,13]\nAD      = df_asset_details.set_index(\"Asset_ID\")\nif test_baseline_model:\n    for x in range(len(id_test)):  \n        y_pred = pd.DataFrame()\n        sample = 15\n\n        curl_models = models[id_test[x]]\n        record      = df_valid[df_valid.Asset_ID == x].head(sample)   \n        y_true      = record['Target']\n        x_test      = get_features(record)\n        rubbish     = gc.collect()\n\n        x_pred = []\n        for i in tqdm(range(x_test.shape[0])):\n            xs_pred = curl_models[2].predict(x_test.iloc[i].to_frame().T)[0][-1][-1]\n            x_pred.append(xs_pred)\n        y_pred['x_pred'] = x_pred\n        print(f\"Test score for LR baseline of {AD.Asset_Name[id_test[x]]:<21}:{np.corrcoef(y_pred.x_pred, y_true)[0,1]:.5f}\")\n        del y_pred","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:27.075599Z","iopub.status.idle":"2022-02-13T17:34:27.076308Z","shell.execute_reply.started":"2022-02-13T17:34:27.076117Z","shell.execute_reply":"2022-02-13T17:34:27.076142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if test_baseline_model:\n    x_pred = pd.DataFrame()\n    for x in tqdm(range(len(df_valid.Asset_ID.unique()))):\n        record           = df_valid[df_valid.Asset_ID == x]     \n        record           = record.drop(['Target','Asset_ID'],axis=1)\n        x_test           = get_features(pd.DataFrame(record))\n        curl_models      = models[x]\n        x_test['y_pred'] = np.mean(np.concatenate([model.predict(x_test)[2][0] for model in curl_models], axis = 0), axis = 0)\n        \n        x_pred           = pd.concat([x_test,x_pred])    \n    \n    x_pred = x_pred.sort_index()    \n    print('Test score for LR baseline: ', f\"{np.corrcoef(x_pred.y_pred,df_valid.Target)[0,1]:.5f}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:27.077392Z","iopub.status.idle":"2022-02-13T17:34:27.077716Z","shell.execute_reply.started":"2022-02-13T17:34:27.077543Z","shell.execute_reply":"2022-02-13T17:34:27.077565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It might not be the best method. However, it may also be used for learning that is as easy to use as the previous version. \nThere should be a more lazy method.","metadata":{}},{"cell_type":"markdown","source":"# LAZY METHOD EXAMPLE","metadata":{}},{"cell_type":"code","source":"def get_model(hp, dim_f, dim_id=1, weight = 1.0, n_asset=14):\n    '''\n    (https://www.kaggle.com/vmuzhichenko/g-research-parallel-lstm-training, https://www.kaggle.com/ysairaajpatro/g-research-lstm-temporal-attention-notebook) + \n    [https://www.kaggle.com/aimind/bottleneck-encoder-mlp-keras-tuner-8601c5/notebook] + \n    [https://www.kaggle.com/lonnieqin/ubiquant-market-prediction-with-dnn]\n    \n    Recommend looking at such an example from to understand how it might be applied\n    '''\n    features_inputs = tf.keras.Input((dim_f, ))\n    asset_id_inputs = tf.keras.Input((dim_id, ))\n    \n    asset_id_x = investment_id_lookup_layer(asset_id_inputs)\n    asset_id_x = layers.Embedding(investment_id_size, n_asset, input_length=1)(asset_id_x)\n    asset_id_x = layers.Reshape((-1, ))(asset_id_x)\n    asset_id_x = layers.Dense(n_asset*3, activation='mish')(asset_id_x)\n\n    x0  = tf.keras.layers.BatchNormalization()(features_inputs)\n    encoder = tf.keras.layers.GaussianNoise(hp.Float(f'noise',0.0,0.5))(x0)\n    encoder = tf.keras.layers.Dense(hp.Int('num_layers_en_0',2**6,2**12))(encoder)\n    encoder = tf.keras.layers.Dense(hp.Int('num_layers_en_1',2**6,2**12))(encoder)\n    encoder = tf.keras.layers.Dense(hp.Int('num_layers_en_2',2**6,2**12))(encoder)\n    encoder = tf.keras.layers.BatchNormalization()(encoder)\n    encoder = tf.keras.layers.Activation('mish')(encoder)\n    \n    decoder = tf.keras.layers.Dropout(hp.Float(f'dropout_de',0.0,0.5))(encoder)\n    decoder = tf.keras.layers.Dense(hp.Int('num_layers_de',2**5, 2**7), name = 'decoder')(decoder)\n    \n    x_ae = tf.keras.layers.Dense(hp.Int('num_layers_ae',2**5, 2**7))(decoder)\n    x_ae = tf.keras.layers.BatchNormalization()(x_ae)\n    x_ae = tf.keras.layers.Activation('mish')(x_ae)\n    x_ae = tf.keras.layers.Dropout(hp.Float(f'dropout_ae',0.0,0.5))(x_ae)\n    \n    out_ae = tf.keras.layers.Dense(1, activation='mish', name = 'ae_out')(x_ae)\n    \n    feature_x = tf.keras.layers.Concatenate()([x0, encoder])\n    feature_x = tf.keras.layers.BatchNormalization()(feature_x)\n    feature_x = tf.keras.layers.Dropout(hp.Float(f'dropout_fx',0.0,0.5))(feature_x)\n\n    x = layers.Concatenate(axis=1)([asset_id_x, feature_x])\n    x = layers.Dense(hp.Int('num_layers_mlp_0',2**5,2**9), activation=\"swish\")(x)\n    x = layers.Dropout(hp.Float(f'dropout_mlp_0',0.0,0.5))(x)\n    x = layers.Dense(hp.Int('num_layers_mlp_1',2**5,2**9), activation=\"swish\")(x)\n    x = layers.Dropout(hp.Float(f'dropout_mlp_1',0.0,0.5))(x)\n    x = layers.Dense(hp.Int('num_layers_,mlp_2',2**5,2**9), activation=\"swish\")(x)\n    x = layers.Dropout(hp.Float(f'dropout_mlp_2',0.0,0.5))(x)\n    \n    output = layers.Dense(1, activation='swish', name = 'output')(x)\n    model = tf.keras.Model(inputs=[investment_id_inputs, features_inputs], outputs=[decoder, out_ae, output])\n    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.01),\n                  loss = {'decoder': tf.keras.losses.MeanAbsoluteError(),\n                          'ae_out' : tf.keras.losses.MeanAbsoluteError(),\n                          'output' : mse_w(weight), \n                         },\n                  metrics = {'decoder': tf.keras.metrics.MeanAbsoluteError(name = 'mae'),\n                             'ae_out' : tf.keras.metrics.MeanAbsoluteError(name = 'mae'),\n                             'output' : tf.keras.metrics.RootMeanSquaredError(name='rmse'),\n                            },\n                 ) \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-02-13T17:34:27.078927Z","iopub.status.idle":"2022-02-13T17:34:27.079298Z","shell.execute_reply.started":"2022-02-13T17:34:27.079121Z","shell.execute_reply":"2022-02-13T17:34:27.079142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# REFERENCE SOURCE CODE\n1. [1st Place of Jane Street ➜ Adapted to Crypto ](https://www.kaggle.com/yamqwe/1st-place-of-jane-street-adapted-to-crypto/notebook#Submit-To-Kaggle-%F0%9F%87%B0)< [Jane Street: Supervised Autoencoder MLP](https://www.kaggle.com/gogo827jz/jane-street-supervised-autoencoder-mlp) < [Bottleneck encoder + MLP + Keras Tuner 8601c5](https://www.kaggle.com/aimind/bottleneck-encoder-mlp-keras-tuner-8601c5/notebook) : [Yam Peleg](https://www.kaggle.com/yamqwe), [Yirun Zhang](https://www.kaggle.com/gogo827jz), [骥](https://www.kaggle.com/aimind), Others","metadata":{}},{"cell_type":"markdown","source":"# PREVIOUS LEVEL\n* [Crypto Forecasting(0/1) : LGBM-QHO : Natapong Nitarach](https://www.kaggle.com/code/natnitarach/crypto-forecasting-0-1-lgbm-qho)","metadata":{}}]}