{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Jane Street 1st place solution -> Crypto","metadata":{}},{"cell_type":"markdown","source":"In this Notebook, I adapt Yirun's 1st place solution from the competion Jane Street to crypto. There is already an excellent kernel made by Yam Peleg with tensorflow, but I decided to make another one using pytorch. In opposite to Yam Peleg's kernel, I am training a single model for all assets.\n\nYirun Zhang's original solution: https://www.kaggle.com/gogo827jz/jane-street-supervised-autoencoder-mlp \n\nYam Peleg's adaptation: https://www.kaggle.com/yamqwe/1st-place-of-jane-street-adapted-to-crypto","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:30.321109Z","iopub.execute_input":"2021-12-07T21:15:30.321940Z","iopub.status.idle":"2021-12-07T21:15:30.345471Z","shell.execute_reply.started":"2021-12-07T21:15:30.321801Z","shell.execute_reply":"2021-12-07T21:15:30.344171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, SequentialSampler\nimport time\nfrom torch.optim import AdamW\nimport warnings\nimport random\nimport os\n\n#!mkdir ../checkpoints","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:30.347113Z","iopub.execute_input":"2021-12-07T21:15:30.348119Z","iopub.status.idle":"2021-12-07T21:15:31.678229Z","shell.execute_reply.started":"2021-12-07T21:15:30.348071Z","shell.execute_reply":"2021-12-07T21:15:31.677161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PurgedGroupTimeSeriesSplit","metadata":{}},{"cell_type":"markdown","source":"See https://www.kaggle.com/yamqwe/purgedgrouptimeseries-cv-with-extra-data-tabnet","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass GroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_size : int, default=None\n        Maximum size for a single training set.\n    Examples\n    --------\n    >>> import numpy as np\n    >>> from sklearn.model_selection import GroupTimeSeriesSplit\n    >>> groups = np.array(['a', 'a', 'a', 'a', 'a', 'a',\\\n                           'b', 'b', 'b', 'b', 'b',\\\n                           'c', 'c', 'c', 'c',\\\n                           'd', 'd', 'd'])\n    >>> gtss = GroupTimeSeriesSplit(n_splits=3)\n    >>> for train_idx, test_idx in gtss.split(groups, groups=groups):\n    ...     print(\"TRAIN:\", train_idx, \"TEST:\", test_idx)\n    ...     print(\"TRAIN GROUP:\", groups[train_idx],\\\n                  \"TEST GROUP:\", groups[test_idx])\n    TRAIN: [0, 1, 2, 3, 4, 5] TEST: [6, 7, 8, 9, 10]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a']\\\n    TEST GROUP: ['b' 'b' 'b' 'b' 'b']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] TEST: [11, 12, 13, 14]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b']\\\n    TEST GROUP: ['c' 'c' 'c' 'c']\n    TRAIN: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]\\\n    TEST: [15, 16, 17]\n    TRAIN GROUP: ['a' 'a' 'a' 'a' 'a' 'a' 'b' 'b' 'b' 'b' 'b' 'c' 'c' 'c' 'c']\\\n    TEST GROUP: ['d' 'd' 'd']\n    \"\"\"\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_size=None\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_size = max_train_size\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n        group_test_size = n_groups // n_folds\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n            for train_group_idx in unique_groups[:group_test_start]:\n                train_array_tmp = group_dict[train_group_idx]\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n            train_end = train_array.size\n            if self.max_train_size and self.max_train_size < train_end:\n                train_array = train_array[train_end -\n                                          self.max_train_size:train_end]\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n            yield [int(i) for i in train_array], [int(i) for i in test_array]\nimport numpy as np\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection._split import _BaseKFold, indexable, _num_samples\nfrom sklearn.utils.validation import _deprecate_positional_args\n\n# modified code for group gaps; source\n# https://github.com/getgaurav2/scikit-learn/blob/d4a3af5cc9da3a76f0266932644b884c99724c57/sklearn/model_selection/_split.py#L2243\nclass PurgedGroupTimeSeriesSplit(_BaseKFold):\n    \"\"\"Time Series cross-validator variant with non-overlapping groups.\n    Allows for a gap in groups to avoid potentially leaking info from\n    train into test if the model has windowed or lag features.\n    Provides train/test indices to split time series data samples\n    that are observed at fixed time intervals according to a\n    third-party provided group.\n    In each split, test indices must be higher than before, and thus shuffling\n    in cross validator is inappropriate.\n    This cross-validation object is a variation of :class:`KFold`.\n    In the kth split, it returns first k folds as train set and the\n    (k+1)th fold as test set.\n    The same group will not appear in two different folds (the number of\n    distinct groups has to be at least equal to the number of folds).\n    Note that unlike standard cross-validation methods, successive\n    training sets are supersets of those that come before them.\n    Read more in the :ref:`User Guide <cross_validation>`.\n    Parameters\n    ----------\n    n_splits : int, default=5\n        Number of splits. Must be at least 2.\n    max_train_group_size : int, default=Inf\n        Maximum group size for a single training set.\n    group_gap : int, default=None\n        Gap between train and test\n    max_test_group_size : int, default=Inf\n        We discard this number of groups from the end of each train split\n    \"\"\"\n\n    @_deprecate_positional_args\n    def __init__(self,\n                 n_splits=5,\n                 *,\n                 max_train_group_size=np.inf,\n                 max_test_group_size=np.inf,\n                 group_gap=None,\n                 verbose=False\n                 ):\n        super().__init__(n_splits, shuffle=False, random_state=None)\n        self.max_train_group_size = max_train_group_size\n        self.group_gap = group_gap\n        self.max_test_group_size = max_test_group_size\n        self.verbose = verbose\n\n    def split(self, X, y=None, groups=None):\n        \"\"\"Generate indices to split data into training and test set.\n        Parameters\n        ----------\n        X : array-like of shape (n_samples, n_features)\n            Training data, where n_samples is the number of samples\n            and n_features is the number of features.\n        y : array-like of shape (n_samples,)\n            Always ignored, exists for compatibility.\n        groups : array-like of shape (n_samples,)\n            Group labels for the samples used while splitting the dataset into\n            train/test set.\n        Yields\n        ------\n        train : ndarray\n            The training set indices for that split.\n        test : ndarray\n            The testing set indices for that split.\n        \"\"\"\n        if groups is None:\n            raise ValueError(\n                \"The 'groups' parameter should not be None\")\n        X, y, groups = indexable(X, y, groups)\n        n_samples = _num_samples(X)\n        n_splits = self.n_splits\n        group_gap = self.group_gap\n        max_test_group_size = self.max_test_group_size\n        max_train_group_size = self.max_train_group_size\n        n_folds = n_splits + 1\n        group_dict = {}\n        u, ind = np.unique(groups, return_index=True)\n        unique_groups = u[np.argsort(ind)]\n        n_samples = _num_samples(X)\n        n_groups = _num_samples(unique_groups)\n        for idx in np.arange(n_samples):\n            if (groups[idx] in group_dict):\n                group_dict[groups[idx]].append(idx)\n            else:\n                group_dict[groups[idx]] = [idx]\n        if n_folds > n_groups:\n            raise ValueError(\n                (\"Cannot have number of folds={0} greater than\"\n                 \" the number of groups={1}\").format(n_folds,\n                                                     n_groups))\n\n        group_test_size = min(n_groups // n_folds, max_test_group_size)\n        group_test_starts = range(n_groups - n_splits * group_test_size,\n                                  n_groups, group_test_size)\n        for group_test_start in group_test_starts:\n            train_array = []\n            test_array = []\n\n            group_st = max(0, group_test_start - group_gap - max_train_group_size)\n            for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:\n                train_array_tmp = group_dict[train_group_idx]\n\n                train_array = np.sort(np.unique(\n                                      np.concatenate((train_array,\n                                                      train_array_tmp)),\n                                      axis=None), axis=None)\n\n            train_end = train_array.size\n\n            for test_group_idx in unique_groups[group_test_start:\n                                                group_test_start +\n                                                group_test_size]:\n                test_array_tmp = group_dict[test_group_idx]\n                test_array = np.sort(np.unique(\n                                              np.concatenate((test_array,\n                                                              test_array_tmp)),\n                                     axis=None), axis=None)\n\n            test_array  = test_array[group_gap:]\n\n\n            if self.verbose > 0:\n                    pass\n\n            yield [int(i) for i in train_array], [int(i) for i in test_array]","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-07T21:15:31.680906Z","iopub.execute_input":"2021-12-07T21:15:31.681629Z","iopub.status.idle":"2021-12-07T21:15:32.460198Z","shell.execute_reply.started":"2021-12-07T21:15:31.681567Z","shell.execute_reply":"2021-12-07T21:15:32.458965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Seed","metadata":{}},{"cell_type":"code","source":"def set_seed(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.465537Z","iopub.execute_input":"2021-12-07T21:15:32.465777Z","iopub.status.idle":"2021-12-07T21:15:32.473532Z","shell.execute_reply.started":"2021-12-07T21:15:32.465746Z","shell.execute_reply":"2021-12-07T21:15:32.472284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"markdown","source":"Kaggle CPU is a bottleneck, so we train our network only on the first fold. If you want to experiment with all folds, then set train_only_first_fold = False.","metadata":{}},{"cell_type":"code","source":"class GlobalConfig:\n    debug = False\n            \n    device = \"cuda\"\n    \n    # PurgedGroupTimeSeriesSplit\n    num_folds = 5\n    group_gap = 31 if not debug else 1\n    train_only_first_fold = True\n    \n    \n    # dataloader\n    train_batch_size = 4096\n    val_batch_size = 4096\n    num_workers = 2\n    \n    \n    # optimizer\n    lr = 1e-2","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.475584Z","iopub.execute_input":"2021-12-07T21:15:32.476526Z","iopub.status.idle":"2021-12-07T21:15:32.485168Z","shell.execute_reply.started":"2021-12-07T21:15:32.476476Z","shell.execute_reply":"2021-12-07T21:15:32.483709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataFrame","metadata":{}},{"cell_type":"code","source":"def create_df():\n    df = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\n    df = df.merge(pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\"), on=\"Asset_ID\", how=\"left\")\n    \n    if GlobalConfig.debug:\n        df = df.iloc[:200000]\n    \n    df = df.set_index(\"timestamp\")\n    \n    df = preprocess_df(df)\n    df = add_row_features(df)\n    df = add_date(df)\n    df = add_group(df)\n    df = df.dropna()\n\n    \n    return df\n\n\n\n\ndef add_row_features(df):\n    \n    df['upper_shadow'] = df['High'] / df[['Close', 'Open']].max(axis=1)\n    df['lower_shadow'] = df[['Close', 'Open']].min(axis=1) / df['Low']\n    df['open2close'] = df['Close'] / df['Open']\n    df['high2low'] = df['High'] / df['Low']\n    mean_price = df[['Open', 'High', 'Low', 'Close']].mean(axis=1)\n    median_price = df[['Open', 'High', 'Low', 'Close']].median(axis=1)\n    df['high2mean'] = df['High'] / mean_price\n    df['low2mean'] = df['Low'] / mean_price\n    df['high2median'] = df['High'] / median_price\n    df['low2median'] = df['Low'] / median_price\n    df['volume2count'] = df['Volume'] / (df['Count'] + 1)\n    \n    return df   \n\n\n\ndef preprocess_df(df):\n    dfs = []\n    df = df.sort_values(by=\"timestamp\")\n    \n    for asset_id in df.Asset_ID.unique():\n        asset_df = df[df.Asset_ID == asset_id]\n        asset_df.reindex(range(asset_df.index[0],asset_df.index[-1]+60,60),method='pad')\n        asset_df = asset_df.replace([np.inf, -np.inf], np.nan)\n        asset_df = asset_df.fillna(method=\"ffill\")  \n\n        dfs.append(asset_df)\n        \n        \n    df = pd.concat(dfs)\n    df = df.sort_values(by=\"timestamp\")\n    \n    return df\n\n\n\ndef add_date(df):\n    df['date'] = pd.to_datetime(df.index, unit='s')        \n    \n    return df\n\n\n\ndef add_group(df):\n    groups = pd.factorize(df['date'].dt.day.astype(str) + '_' + df['date'].dt.month.astype(str) +\\\n                                                          '_' + df['date'].dt.year.astype(str))[0]\n    \n    df[\"group\"] = groups\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.487283Z","iopub.execute_input":"2021-12-07T21:15:32.488116Z","iopub.status.idle":"2021-12-07T21:15:32.510225Z","shell.execute_reply.started":"2021-12-07T21:15:32.488066Z","shell.execute_reply":"2021-12-07T21:15:32.508979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CryptoDataset","metadata":{}},{"cell_type":"code","source":"class CryptoDataset:\n    def __init__(self, df, is_train=True):\n        self.df = df\n        self.is_train = is_train\n        self.features = ['Asset_ID', 'upper_shadow', 'lower_shadow', 'open2close', 'high2low', 'high2mean', 'low2mean',\\\n                                         'high2median', 'low2median', 'volume2count']\n        \n        \n    def __len__(self):\n        return self.df.shape[0]\n    \n    \n    \n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        \n        features = row[self.features].values.astype(np.float32)\n        weight = row[\"Weight\"].astype(np.float32)\n        \n        if self.is_train:\n            target = row[\"Target\"].astype(np.float32)\n            return {\"features\" : features, \"target\" : target, \"weight\" : weight}\n        \n        else:\n            return {\"features\" : features, \"weight\" : weight}","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.512455Z","iopub.execute_input":"2021-12-07T21:15:32.513289Z","iopub.status.idle":"2021-12-07T21:15:32.526036Z","shell.execute_reply.started":"2021-12-07T21:15:32.513224Z","shell.execute_reply":"2021-12-07T21:15:32.524621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DataLoader","metadata":{}},{"cell_type":"code","source":"def get_dataloader(df, batch_size, is_train=True):\n    crypto_dataset = CryptoDataset(df, is_train=is_train)\n    dataloader = DataLoader(crypto_dataset, sampler=SequentialSampler(crypto_dataset), batch_size=batch_size, pin_memory=True, num_workers=GlobalConfig.num_workers)\n    \n    return dataloader","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.528096Z","iopub.execute_input":"2021-12-07T21:15:32.529285Z","iopub.status.idle":"2021-12-07T21:15:32.541251Z","shell.execute_reply.started":"2021-12-07T21:15:32.529233Z","shell.execute_reply":"2021-12-07T21:15:32.540070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Architectures","metadata":{}},{"cell_type":"code","source":"def make_model():\n    \n    # Preprocessing Layer\n    num_features = 10\n    num_embeddings = 14\n    embedding_dim = 10\n    \n    \n    \n    # Encoder\n    encoder_input = num_features + embedding_dim - 1\n    encoder_output = int((num_features + embedding_dim - 1) * 0.8)\n    \n    \n    \n    # Decoder\n    decoder_input = encoder_output\n    decoder_hidden = 80\n    decoder_output = num_features - 1\n    \n    \n    \n    # Head Decoder\n    head_decoder_input = decoder_output\n    head_decoder_hidden = int(decoder_output * 1.5)\n\n    \n    # FFN\n    ffn_input = encoder_input + decoder_input \n    num_blocks = 2\n    \n    \n    preprocessing_layer = PreprocessingLayer(num_features, num_embeddings, embedding_dim)\n    encoder = Encoder(encoder_input, encoder_output)\n    decoder = Decoder(decoder_input, decoder_hidden, decoder_output)\n    head_decoder = Head(head_decoder_input, head_decoder_hidden)\n    ffn = FFNetwork(ffn_input, num_blocks)\n\n    \n    main_network = MainNetwork(preprocessing_layer, encoder, decoder, head_decoder, ffn)\n    \n    return main_network","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.543452Z","iopub.execute_input":"2021-12-07T21:15:32.544356Z","iopub.status.idle":"2021-12-07T21:15:32.555596Z","shell.execute_reply.started":"2021-12-07T21:15:32.544202Z","shell.execute_reply":"2021-12-07T21:15:32.554281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Main Network","metadata":{}},{"cell_type":"code","source":"class MainNetwork(nn.Module):\n    def __init__(self, preprocessing_layer, encoder, decoder, head_decoder, ffn):\n        super(MainNetwork, self).__init__()\n        \n        self.preprocessing_layer = preprocessing_layer\n        \n        self.encoder = encoder\n        self.decoder = decoder\n        self.head_decoder = head_decoder\n\n        self.ffn = ffn\n        \n    \n        \n        \n    def forward(self, x):\n        x = self.preprocessing_layer(x)\n        \n        out_encoder = self.encoder(x)\n        \n        target_predictions = self.ffn(torch.cat([x, out_encoder], dim=1) )\n        features_reconstruction = self.decoder(out_encoder)\n        target_predictions_head_decoder = self.head_decoder(features_reconstruction)\n        \n        \n        return target_predictions, target_predictions_head_decoder, features_reconstruction\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.557416Z","iopub.execute_input":"2021-12-07T21:15:32.558146Z","iopub.status.idle":"2021-12-07T21:15:32.571568Z","shell.execute_reply.started":"2021-12-07T21:15:32.558096Z","shell.execute_reply":"2021-12-07T21:15:32.570375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Encoder","metadata":{}},{"cell_type":"code","source":"class Encoder(nn.Module): \n    def __init__(self, num_input_features, num_output_features):\n        super(Encoder, self).__init__()\n        \n        self.fcl = nn.Linear(num_input_features, num_output_features)\n        self.bn = nn.BatchNorm1d(num_output_features)\n        \n\n    def forward(self, x):\n        #x = add_gaussian_noise(x)\n        x = self.fcl(x)\n        x = swish(self.bn(x))\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.573833Z","iopub.execute_input":"2021-12-07T21:15:32.574805Z","iopub.status.idle":"2021-12-07T21:15:32.584637Z","shell.execute_reply.started":"2021-12-07T21:15:32.574752Z","shell.execute_reply":"2021-12-07T21:15:32.583395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Decoder","metadata":{}},{"cell_type":"code","source":"class Decoder(nn.Module):  \n    def __init__(self, num_input_features, num_hidden_units, num_output_features):\n        super(Decoder, self).__init__()\n        self.dropout = nn.Dropout(p=0.1)\n        \n        \n        #self.fcl = nn.Linear(num_input_features, num_output_features) \n    \n        self.fcl = nn.Linear(num_input_features, num_hidden_units)\n        self.fcl_output = nn.Linear(num_hidden_units, num_output_features)\n        \n    def forward(self, x):\n        x = self.dropout(x)\n        #x = self.fcl(x)\n        x = swish(self.fcl(x))\n        x = self.fcl_output(x)\n        \n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.586739Z","iopub.execute_input":"2021-12-07T21:15:32.587540Z","iopub.status.idle":"2021-12-07T21:15:32.598468Z","shell.execute_reply.started":"2021-12-07T21:15:32.587490Z","shell.execute_reply":"2021-12-07T21:15:32.597397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Head (decoder)","metadata":{}},{"cell_type":"code","source":"class Head(nn.Module): \n    def __init__(self, num_input_features, num_hidden_units):\n        super(Head, self).__init__()\n        self.fcl = nn.Linear(num_input_features, num_hidden_units)\n        self.bn = nn.BatchNorm1d(num_hidden_units)\n        self.dropout = nn.Dropout(p=0.5)\n        self.fcl_output = nn.Linear(num_hidden_units, 1)\n        \n\n    \n    \n    def forward(self, x):\n        x = self.fcl(x)\n        x = swish(self.bn(x))\n        x = self.fcl_output(self.dropout(x))            \n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.602992Z","iopub.execute_input":"2021-12-07T21:15:32.604102Z","iopub.status.idle":"2021-12-07T21:15:32.614057Z","shell.execute_reply.started":"2021-12-07T21:15:32.604049Z","shell.execute_reply":"2021-12-07T21:15:32.612605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### FFNetwork","metadata":{}},{"cell_type":"code","source":"class FFNetwork(nn.Module): \n    def __init__(self, num_hidden_units, num_blocks):\n        super(FFNetwork, self).__init__()\n        self.bn = nn.BatchNorm1d(num_hidden_units)\n        self.dropout = nn.Dropout(p=0.5)\n        self.backbone = nn.Sequential(*[BlockFFNetwork(num_hidden_units) for i in range(num_blocks)])\n        \n        self.fcl_output = nn.Linear(num_hidden_units, 1)\n        \n        \n    def forward(self, x):\n        x = self.dropout(self.bn(x))\n        x = self.backbone(x)\n        x = self.fcl_output(x)\n        \n        return x\n\n    \n    \n        \nclass BlockFFNetwork(nn.Module):\n    def __init__(self, num_hidden_units):\n        super(BlockFFNetwork, self).__init__()\n        \n        self.fcl = nn.Linear(num_hidden_units, num_hidden_units)\n        self.bn = nn.BatchNorm1d(num_hidden_units)\n        \n        self.dropout = nn.Dropout(p=0.5)\n\n    \n    \n    def forward(self, x):\n        x = self.fcl(x)\n        x = swish(self.bn(x))\n        x = self.dropout(x)\n               \n        return x","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.617205Z","iopub.execute_input":"2021-12-07T21:15:32.618492Z","iopub.status.idle":"2021-12-07T21:15:32.631811Z","shell.execute_reply.started":"2021-12-07T21:15:32.618439Z","shell.execute_reply":"2021-12-07T21:15:32.630683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Preprocessing Layer","metadata":{}},{"cell_type":"code","source":"class PreprocessingLayer(nn.Module): \n    def __init__(self, num_features, num_embeddings, embedding_dim=10):\n        super(PreprocessingLayer, self).__init__()\n        self.embedding = nn.Embedding(num_embeddings, embedding_dim)\n        self.bn = nn.BatchNorm1d(num_features + embedding_dim - 1)\n                    \n            \n    def forward(self, x):\n        \n        ids = x[:,0].long()\n        remaining_features = x[:, 1:]\n        \n        embedded_ids = self.embedding(ids)\n        \n        output = self.bn(torch.cat([embedded_ids, remaining_features], dim=1))\n        \n        return output","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.634529Z","iopub.execute_input":"2021-12-07T21:15:32.635799Z","iopub.status.idle":"2021-12-07T21:15:32.646722Z","shell.execute_reply.started":"2021-12-07T21:15:32.635750Z","shell.execute_reply":"2021-12-07T21:15:32.645606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Helper functions","metadata":{}},{"cell_type":"code","source":"def add_gaussian_noise(x):              \n    return x + torch.randn(*x.shape).to(GlobalConfig.device)         #naive implementation, since different features have different scale\n    \n    \ndef swish(x):        \n    return x * torch.sigmoid(x)","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.649780Z","iopub.execute_input":"2021-12-07T21:15:32.650304Z","iopub.status.idle":"2021-12-07T21:15:32.657531Z","shell.execute_reply.started":"2021-12-07T21:15:32.650193Z","shell.execute_reply":"2021-12-07T21:15:32.656344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Metrics","metadata":{}},{"cell_type":"markdown","source":"I took the weighted correlation function from https://www.kaggle.com/code1110/gresearch-simple-lgb-starter\n\nAverageMeter and CorrelationMeter are inspired from the work of Alex Shonenkov, see https://www.kaggle.com/shonenkov/training-cv-melanoma-starter.","metadata":{}},{"cell_type":"code","source":"def corr(x, y, w):\n    \"\"\"Weighted Correlation\"\"\"\n    return cov(x, y, w) / np.sqrt(cov(x, x, w) * cov(y, y, w))\n\ndef cov(x, y, w):\n    \"\"\"Weighted Covariance\"\"\"\n    return np.sum(w * (x - m(x, w)) * (y - m(y, w))) / np.sum(w)\n\ndef m(x, w):\n    \"\"\"Weighted Mean\"\"\"\n\n    return np.sum(x * w) / np.sum(w)\n\n\n\n\ndef corr_torch(x, y, w):\n    \"\"\"Weighted Correlation\"\"\"\n    return cov_torch(x, y, w) / torch.sqrt(cov_torch(x, x, w) * cov_torch(y, y, w))\n\ndef cov_torch(x, y, w):\n    \"\"\"Weighted Covariance\"\"\"\n    return torch.sum(w * (x - m_torch(x, w)) * (y - m_torch(y, w))) / torch.sum(w)\n\n\ndef m_torch(x, w):\n    \"\"\"Weighted Mean\"\"\"\n    return torch.sum(x * w) / torch.sum(w)\n\n\n\n\nclass AverageMeter:\n    def __init__(self):\n        self.reset()\n\n        \n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n        \n        \n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        \n        self.avg = self.sum / self.count\n        \n        \n        \n    \nclass CorrelationMeter:\n    def __init__(self):\n        self.reset()\n        \n        \n    \n    def reset(self):\n        self.predictions = np.array([])\n        self.targets = np.array([])\n        self.weights = np.array([])\n        self.avg = 0\n        \n        \n    \n    def update(self, y_true, y_pred, weights):\n        self.predictions = np.hstack([self.predictions, y_pred.detach().cpu().squeeze().numpy()])\n        self.targets = np.hstack([self.targets, y_true.cpu().numpy()])\n        self.weights = np.hstack([self.weights, weights.cpu().numpy()])\n        \n        \n        self.avg = corr(self.predictions, self.targets, self.weights)","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.659993Z","iopub.execute_input":"2021-12-07T21:15:32.660976Z","iopub.status.idle":"2021-12-07T21:15:32.681545Z","shell.execute_reply.started":"2021-12-07T21:15:32.660922Z","shell.execute_reply":"2021-12-07T21:15:32.680008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss ","metadata":{}},{"cell_type":"code","source":"def criterion(target_predictions, target_predictions_head_decoder, targets,\\\n                                                            features_reconstruction, features, weights):\n    \n    return -(corr_torch(target_predictions.squeeze(), targets, weights) +\\\n             corr_torch(target_predictions_head_decoder.squeeze(), targets, weights))  \n\n    #return F.mse_loss(target_predictions.squeeze(), targets) + F.mse_loss(target_predictions_head_decoder.squeeze(), targets)+\\\n        #F.mse_loss( features_reconstruction, features[:,1:] )","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.682762Z","iopub.execute_input":"2021-12-07T21:15:32.683093Z","iopub.status.idle":"2021-12-07T21:15:32.695218Z","shell.execute_reply.started":"2021-12-07T21:15:32.683047Z","shell.execute_reply":"2021-12-07T21:15:32.694083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train function","metadata":{}},{"cell_type":"markdown","source":"Training function inspired from https://www.kaggle.com/shonenkov/training-cv-melanoma-starter","metadata":{}},{"cell_type":"code","source":"#return target_predictions, target_predictions_head_decoder, features_reconstruction\ndef train(df, num_epochs=10):\n    \n    groups = df[\"group\"].values\n    \n    for fold, (train_idx, val_idx) in \\\n                    enumerate(PurgedGroupTimeSeriesSplit(n_splits=GlobalConfig.num_folds,\\\n                            group_gap=GlobalConfig.group_gap).split(df, y=None, groups=groups)):\n\n        if GlobalConfig.train_only_first_fold and fold > 0:\n            break\n            \n            \n        if not os.path.exists(f\"../working/checkpoints/fold_{fold}\"):\n            os.makedirs(f\"../working/checkpoints/fold_{fold}\")\n        \n        print(\"% of data used for training: \", len(train_idx)/df.shape[0] * 100, \"% of data used for validation: \",  len(val_idx)/df.shape[0] * 100)    \n        \n        model = make_model().to(GlobalConfig.device)\n        optimizer = AdamW(model.parameters(), lr=GlobalConfig.lr)\n        \n        \n        \n        train_loader = get_dataloader(df.iloc[train_idx], batch_size=GlobalConfig.train_batch_size, is_train=True)\n        val_loader = get_dataloader(df.iloc[val_idx], batch_size=GlobalConfig.val_batch_size, is_train=True)\n        \n        \n        best_final_scores = -10\n        \n        for epoch in range(num_epochs):\n\n            t = time.time()\n            summary_loss_train, final_scores_train = train_one_epoch(model, optimizer, train_loader)\n            print(f'[RESULT]: Train. Epoch: {epoch}, summary_loss: {summary_loss_train.avg:.5f}, final_score: {final_scores_train.avg:.5f}, time: {(time.time() - t):.5f}')\n\n            \n            \n            summary_loss_val, final_scores_val = validation_one_epoch(model, val_loader)\n            print(f'[RESULT]: Val. Epoch: {epoch}, summary_loss: {summary_loss_val.avg:.5f}, final_score: {final_scores_val.avg:.5f}')\n    \n            \n        \n            model.eval()\n\n            \n            save_checkpoint(model, optimizer, fold, name=\"last_checkpoint\")\n                        \n            if final_scores_val.avg > best_final_scores: \n                best_final_scores = final_scores_val.avg\n                save_checkpoint(model, optimizer, fold, name=f\"score_{best_final_scores}\")\n                \n\n        \n        \n        \n\n        \n    \ndef train_one_epoch(model, optimizer, train_loader):\n    \n    model.train()        \n    summary_loss = AverageMeter()\n    final_scores = CorrelationMeter()\n    t = time.time()\n    \n    for step, batch in enumerate(train_loader):\n        if  (step % 2 == 0):\n            print(\n                f'Train Step {step}/{len(train_loader)}, ' + \\\n                f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \\\n                f'time: {(time.time() - t):.5f}', end='\\r'\n                )\n            \n       \n        weights = batch[\"weight\"].to(GlobalConfig.device).float()\n        features = batch[\"features\"].to(GlobalConfig.device).float()\n        targets = batch[\"target\"].to(GlobalConfig.device).float()\n\n                \n        batch_size = weights.shape[0]\n                      \n \n        optimizer.zero_grad()     \n\n        target_predictions, target_predictions_head_decoder, features_reconstruction = model(features)\n        \n        \n        \n        \n        loss = criterion(target_predictions, target_predictions_head_decoder, targets,\\\n                                                            features_reconstruction, features, weights)\n\n        loss.backward()\n        \n        \n        \n        \n        summary_loss.update(loss.detach().cpu().item(), batch_size)\n        final_scores.update(targets, target_predictions, weights)\n            \n        optimizer.step()\n\n            \n                            \n    return summary_loss, final_scores\n\n\n\n\ndef validation_one_epoch(model, val_loader):\n        model.eval()\n        \n        summary_loss = AverageMeter()\n        final_scores = CorrelationMeter()\n\n        for step, batch in enumerate(val_loader):\n            \n            if step % 10 == 0:\n                print(\n                    f'Val Step {step}/{len(val_loader)}, ' + \\\n                    f'summary_loss: {summary_loss.avg:.5f}, final_score: {final_scores.avg:.5f} ', end='\\r')\n\n            \n            \n            weights = batch[\"weight\"].to(GlobalConfig.device).float()\n            features = batch[\"features\"].to(GlobalConfig.device).float()\n            targets = batch[\"target\"].to(GlobalConfig.device).float()\n\n                            \n            batch_size = weights.shape[0]\n                      \n             \n   \n            with torch.no_grad():\n                target_predictions, target_predictions_head_decoder, features_reconstruction = model(features)\n\n            \n            loss = criterion(target_predictions, target_predictions_head_decoder, targets,\\\n                                                            features_reconstruction, features, weights)\n\n            \n            summary_loss.update(loss.cpu().item(), batch_size)\n            final_scores.update(targets, target_predictions, weights)\n            \n                \n                \n            \n        return summary_loss, final_scores\n        \n        \n        \ndef save_checkpoint(model, optimizer, fold, name):\n    checkpoint = { \"model_state_dict\" : model.state_dict(), \"optimizer_state_dict\" : optimizer.state_dict()}\n    torch.save(checkpoint, f\"../working/checkpoints/fold_{fold}/{name}.pt\")\n    \n    \n    \n    \ndef load_best_checkpoint(model, fold):\n    name = sorted(os.listdir( f\"../working/checkpoints/fold_{fold}\" ),\\\n                  key = lambda name : name.split(\"_\")[-1].split(\".\"), reverse=True)[0]\n    checkpoint = torch.load( f\"../working/checkpoints/fold_{fold}/{name}.pt\" )\n    \n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    \n    \n    \ndef load_last_checkpoint(model, optimizer, fold):\n    checkpoint = torch.load( f\"../working/checkpoints/fold_{fold}/last_checkpoint.pt\" )\n    \n    model.load_state_dict(checkpoint[\"model_state_dict\"])\n    optimizer.load_state_dict(checkpoint[\"optimizer_state_dict\"])","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.697792Z","iopub.execute_input":"2021-12-07T21:15:32.698426Z","iopub.status.idle":"2021-12-07T21:15:32.733089Z","shell.execute_reply.started":"2021-12-07T21:15:32.698375Z","shell.execute_reply":"2021-12-07T21:15:32.731743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"set_seed()\ndf = create_df()\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:15:32.734722Z","iopub.execute_input":"2021-12-07T21:15:32.735007Z","iopub.status.idle":"2021-12-07T21:19:21.928290Z","shell.execute_reply.started":"2021-12-07T21:15:32.734946Z","shell.execute_reply":"2021-12-07T21:19:21.926913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train(df, num_epochs=10)","metadata":{"execution":{"iopub.status.busy":"2021-12-07T21:19:21.930204Z","iopub.execute_input":"2021-12-07T21:19:21.930648Z","iopub.status.idle":"2021-12-07T21:32:55.515939Z","shell.execute_reply.started":"2021-12-07T21:19:21.930599Z","shell.execute_reply":"2021-12-07T21:32:55.514292Z"},"trusted":true},"execution_count":null,"outputs":[]}]}