{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\n\n# %% import\nimport os\nimport warnings\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\nwarnings.simplefilter('ignore')\nimport gc\nimport pandas as pd\nimport numpy as np\nimport joblib\nfrom sklearn.model_selection import KFold\nimport scipy\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nimport tensorflow.keras.backend as K\nfrom tensorflow.python.framework.ops import reset_default_graph\n\n\n# %% cfg\nclass Config:\n    plat = 'kaggle' # train on the local platform: local    test: kaggle\n    path = '/input/child-mind-institute-detect-sleep-states'\n    path2 = '/input/model202309231'\n    if plat=='kaggle':\n        path = '/kaggle' + path\n        path2 = '/kaggle' + path2\n    elif plat=='local':\n        path = '.' + path\n        path2 = '.' + path2\n    model = 'CNN'\n    seed = 42\n    anglezmax = 90\n    timegap = 5\n    w_l = round(12*60*12)\n    w_s = round(1 *60*12)\n    w_t = round(3 *60*12)\n    maxnum = 200\n    KFold = 5\n    lr_decay = 10\n    early_stopping = 20\n    learning_rate = 1e-3\n    batch_size = 64\n    epochs = 200\n    closing_size = round(1.5*60*12)\n    smoothing_length = round(0.5*60*12)\n    normalization = False\n    index = ['anglemax', 'nemo', 'timestamp']\ncfg = Config()\nnp.random.seed(cfg.seed)\nos.environ['PYTHONHASHSEED'] = str(cfg.seed)\nos.environ['TF_DETERMINISTIC_OPS'] = '1'\ntf.random.set_seed(cfg.seed)\ntf.keras.utils.set_random_seed(cfg.seed)\ntf.config.experimental.enable_op_determinism()\n\n\n#%% preprocessdata\ndef preprocessdata():\n    label_train = pd.read_csv(os.path.join(cfg.path, 'train_events.csv'))\n    series = np.unique(label_train.series_id)\n    data_train = pd.read_parquet(os.path.join(cfg.path, 'train_series.parquet'))\n    for i, serie in enumerate(series):\n        print(f'{serie} {i+1}/{len(series)}')\n        data_temp = data_train[data_train.series_id==serie].reset_index(drop=True)\n        \n        # data_temp['timestamp'] = pd.to_datetime(data_temp['timestamp']).apply(lambda t: t.tz_localize(None))\n        # data_temp['timestamp'] = data_temp['timestamp'].dt.hour + data_temp['timestamp'].dt.minute / 60\n        data_temp['timestamp'] = data_temp.timestamp.str.split('T', expand=True).loc[:, 1].str.split(':', expand=True).loc[:, 0]\n        data_temp['timestamp'] = data_temp['timestamp'].astype(int)\n        \n        label_temp = label_train[label_train.series_id==serie].reset_index(drop=True)\n        index = np.arange(0, max(data_temp.step)-cfg.w_l, cfg.w_s)\n        label_save = np.zeros((len(data_temp)))\n        for j in range(len(label_temp)):\n            if label_temp.step[j] > 0:\n                if label_temp.event[j] == 'onset':\n                    label_save[round(label_temp.step[j]):] = 1\n                elif label_temp.event[j] == 'wakeup':\n                    label_save[round(label_temp.step[j]):] = 0\n                else:\n                    assert 0==1\n            else:\n                if j==0:\n                    label_save[0:] = -1\n                else:\n                    if max(label_temp.step[:j]) >= 0:\n                        label_save[round(max(label_temp.step[:j])):] = -1\n                    else:\n                        label_save[0:] = -1\n        data_save = np.zeros((len(index), cfg.w_l, 3))\n        label_save2 = np.zeros((len(index), cfg.w_l))\n        for j in range(len(index)):\n            data_save[j, :, 0] = data_temp.anglez[index[j]:(index[j] + cfg.w_l)]\n            data_save[j, :, 1] = data_temp.enmo[index[j]:(index[j] + cfg.w_l)]\n            data_save[j, :, 2] = data_temp.timestamp[index[j]:(index[j] + cfg.w_l)]\n            label_save2[j, :]  = label_save[index[j]:(index[j] + cfg.w_l)]\n        del data_temp, label_save\n        select = np.sum(label_save2==-1, 1) <= (label_save2.shape[1]/2)\n        data_save = data_save[select]\n        label_save2 = label_save2[select]\n        label_save2[label_save2==-1] = 0\n        \n        select = (np.sum(label_save2, 1) > 0) * (np.sum(label_save2, 1) < label_save2.shape[1])\n        data_save1 = data_save[select]\n        data_save2 = data_save[~select]\n        label_save1 = label_save2[select]\n        label_save2 = label_save2[~select]\n        r = np.random.permutation(data_save2.shape[0])\n        data_save2 = data_save2[r][:data_save1.shape[0]]\n        label_save2 = label_save2[r][:data_save1.shape[0]]\n        data_save = np.concatenate((data_save1, data_save2), 0)\n        label_save = np.concatenate((label_save1, label_save2), 0)\n        r = np.random.permutation(data_save.shape[0])\n        data_save = data_save[r][:cfg.maxnum]\n        label_save = label_save[r][:cfg.maxnum]\n        if not os.path.exists('data'):\n            os.makedirs('data')\n        joblib.dump({'data': data_save, 'label': label_save}, os.path.join('data', f'{serie}_{data_save.shape[0]}.pkl'))\n\n\n#%% getinfo\ndef getinfo(flat):\n    if flat:\n        dirs = os.listdir('data')\n        dirs = pd.DataFrame({'path': dirs, 'serie': dirs, 'num': 0, 'fold': 0})\n        for i, path in enumerate(dirs.path):\n            dirs.loc[i, 'serie'] = dirs.path[i].split('_')[0]\n            dirs.loc[i, 'num'] = int(dirs.path[i].split('_')[1].split('.')[0])\n        sgkf = KFold(n_splits=cfg.KFold, random_state=cfg.seed, shuffle=True)\n        for i, (train_index, test_index) in enumerate(sgkf.split(dirs.index, groups=dirs.serie)):\n            dirs.loc[test_index, 'fold'] = i + 1\n            print(f'fold {i+1}  train&valid series number: {sum(dirs.fold!=(i+1))}' +\n                  f'  test series number: {sum(dirs.fold==(i+1))}')\n        dirs.to_csv('info.csv', index=False)\n    else:\n        dirs = pd.read_csv('info.csv')\n    dirs.num[dirs.num>cfg.maxnum] = cfg.maxnum\n    return dirs\n\n\n#%% loaddata\ndef loaddata(trainvalid, fold, dirs):\n    if trainvalid == 'train':\n        dirs_temp = dirs[dirs.fold!=fold].reset_index(drop=True)\n    elif trainvalid == 'valid':\n        dirs_temp = dirs[dirs.fold==fold].reset_index(drop=True)\n    data_train = np.zeros((sum(dirs_temp.num), cfg.w_l, len(cfg.index)))\n    label_train = np.zeros((sum(dirs_temp.num), cfg.w_l))\n    num = 0\n    for j, path in enumerate(dirs_temp.path):\n        if dirs_temp.num[j]>0:\n            data = joblib.load(os.path.join('data', path))\n            data_temp = data['data']\n            label_temp = data['label']\n            data_temp = data_temp[:cfg.maxnum, :]\n            label_temp = label_temp[:cfg.maxnum, :]\n            if cfg.normalization:\n                data_temp = (data_temp - np.mean(data_temp, axis=1, keepdims=True)) / (np.std(data_temp, axis=1, keepdims=True) + 1e-6)\n            else:\n                data_temp[:, :, 0] = data_temp[:, :, 0] / cfg.anglezmax\n                data_temp[:, :, 2] = data_temp[:, :, 2] / 24\n            data_train[num:(num+dirs_temp.num[j]), :, :] = data_temp\n            label_train[num:(num+dirs_temp.num[j]), :] = label_temp\n            num = num + dirs_temp.num[j]\n    r = np.random.permutation(data_train.shape[0])\n    data_train = data_train[r]\n    label_train = label_train[r]\n    return data_train, label_train\n\n\n#%% model select\nclass TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, feat_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)\n        self.ffn = tf.keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"gelu\"), layers.Dense(feat_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output, weights = self.att(inputs, inputs, return_attention_scores=True)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output), weights\n\nclass ClassToken(layers.Layer):\n    \"\"\"Append a class token to an input layer.\"\"\"\n    def build(self, input_shape):\n        cls_init = tf.zeros_initializer()\n        self.hidden_size = input_shape[-1]\n        self.cls = tf.Variable(\n            name=\"cls\",\n            initial_value=cls_init(shape=(1, 1, self.hidden_size), dtype=\"float32\"),\n            trainable=True,\n        )\n    def call(self, inputs):\n        batch_size = tf.shape(inputs)[0]\n        cls_broadcasted = tf.cast(\n            tf.broadcast_to(self.cls, [batch_size, 1, self.hidden_size]),\n            dtype=inputs.dtype,\n        )\n        return tf.concat([cls_broadcasted, inputs], 1)\n\ndef layer_rename(x, name):\n    x = layers.Lambda(lambda x:x, name=name)(x)\n    return x\n\ndef CNN(timesteps: int, classes: int, inputshape:int):\n    ## input ##\n    kernal_num = 32\n    x = layers.Input(shape=(timesteps, inputshape))\n    \n    y1 = layers.Conv1D(kernal_num, kernel_size=16, strides=1, padding='same', activation='gelu')(x)\n    y1 = layers.BatchNormalization()(y1)\n    y1 = layers.MaxPool1D(2)(y1)\n    \n    y2 = layers.Conv1D(kernal_num*2, kernel_size=16, strides=1, padding='same', activation='gelu')(y1)\n    y2 = layers.BatchNormalization()(y2)\n    y2 = layers.MaxPool1D(4)(y2)\n    \n    y3 = layers.Conv1D(kernal_num*4, kernel_size=16, strides=1, padding='same', activation='gelu')(y2)\n    y3 = layers.BatchNormalization()(y3)\n    y3 = layers.MaxPool1D(4)(y3)\n    \n    y4 = layers.Conv1D(kernal_num*4, kernel_size=16, strides=1, padding='same', activation='gelu')(y3)\n    y4 = layers.BatchNormalization()(y4)\n    y4 = layers.MaxPool1D(3)(y4)\n    \n    y4 = ClassToken()(y4)\n    transformer_block = TransformerBlock(y4.shape[2], y4.shape[2], 3, y4.shape[2], 0.1)\n    y4, weights = transformer_block(y4)\n    \n    transformer_block = TransformerBlock(y4.shape[2], y4.shape[2], 3, y4.shape[2], 0.1)\n    y5, weights = transformer_block(y4)\n    \n    transformer_block = TransformerBlock(y5.shape[2], y5.shape[2], 3, y5.shape[2], 0.1)\n    y4, weights = transformer_block(y5)\n\n    y3_T = layers.Conv1DTranspose(kernal_num*4, kernel_size=16, strides=3, padding='same', activation='gelu')(layers.Lambda(lambda v: v[:, 1:, :])(y4))\n    y3_T = y3_T + y3\n    y3_T = layers.BatchNormalization()(y3_T)\n    \n    y2_T = layers.Conv1DTranspose(kernal_num*2, kernel_size=16, strides=4, padding='same', activation='gelu')(y3_T)\n    y2_T = y2_T + y2\n    y2_T = layers.BatchNormalization()(y2_T)\n    \n    y1_T = layers.Conv1DTranspose(kernal_num, kernel_size=16, strides=4, padding='same', activation='gelu')(y2_T)\n    y1_T = y1_T + y1\n    y1_T = layers.BatchNormalization()(y1_T)\n    \n    out1 = layers.Conv1DTranspose(1, kernel_size=16, strides=2, padding='same', activation='sigmoid')(y1_T)\n    out1 = tf.reduce_mean(out1, axis=2)\n    out1 = layer_rename(out1, 'out1')\n    \n    # out2 = layers.Flatten()(y4)\n    out2 = layers.Lambda(lambda v: v[:, 0, :])(y5)\n    # out2 = layers.Dense(32, activation='gelu')(out2)\n    out2 = layers.Dense(classes, activation='sigmoid', name='out2')(out2)\n    \n    return tf.keras.models.Model(inputs=x, outputs=[out1, out2])\n\n\ndef model_select(name: str, timesteps: int, classes: int, inputshape: int):\n    print('model: ' + name)\n    if name   == 'CNN':\n        model = CNN(timesteps=timesteps, classes=classes, inputshape=inputshape)\n    return model\n\n\n#%% loss\ndef loss1(y_true, y_pred):\n    # loss = tf.abs(y_true - y_pred)\n    # loss = tf.reduce_mean(loss)\n    y_true = tf.reshape(y_true, (-1, 1))\n    y_pred = tf.reshape(y_pred, (-1, 1))\n    loss = tf.losses.binary_crossentropy(y_true, y_pred)\n    \n    return loss\n\ndef loss2(y_true, y_pred):\n    loss = tf.losses.binary_crossentropy(y_true, y_pred)\n    return loss\n\n\n#%% train\ndef train(dirs):\n    for i in range(cfg.KFold):\n        print(f'fold {i+1}')\n        data_train, label_train = loaddata('train', i+1, dirs)\n        data_valid, label_valid = loaddata('valid', i+1, dirs)\n        label_train2 = np.sum(label_train, 1) / label_train.shape[1]\n        label_valid2 = np.sum(label_valid, 1) / label_valid.shape[1]\n        label_train = {'out1': label_train, 'out2': label_train2}\n        label_valid = {'out1': label_valid, 'out2': label_valid2}\n        loss_weights = {'out1': 1, 'out2': 1}\n        model_save_path = os.path.join(cfg.path2, cfg.model)\n        callbacks_list = [tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=cfg.lr_decay),\n                          tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join(model_save_path, f'fold{i+1}.h5'),\n                                                             monitor='val_loss', mode='min',\n                                                             save_weights_only=True, save_best_only=True),\n                          tf.keras.callbacks.EarlyStopping(patience=cfg.early_stopping, monitor='val_loss', mode='min')]\n        model = model_select(cfg.model, data_train.shape[1], 1, len(cfg.index))\n        model.compile(optimizer=tf.keras.optimizers.Adam(cfg.learning_rate),\n                      loss={'out1': loss1, 'out2': loss2},\n                      loss_weights=loss_weights)\n        if not os.path.exists(model_save_path):\n            os.makedirs(model_save_path)\n        history = model.fit(data_train, label_train, batch_size=cfg.batch_size, epochs=cfg.epochs,\n                            verbose=1, callbacks=callbacks_list, validation_data=(data_valid, label_valid))\n        \n        # model.load_weights(os.path.join(model_save_path, f'fold{i+1}.h5'))\n        # result = model.predict(data_valid)\n        # result1 = result[0]\n        # result2 = result[1]\n        \n        del history, model\n        gc.collect()\n        tf.keras.backend.clear_session()\n        K.clear_session()\n        reset_default_graph()\n        print('='*50)\n\n\ndef test():\n    model_all = list()\n    for i in range(cfg.KFold):\n        model = model_select(cfg.model, cfg.w_l, 1, len(cfg.index))\n        model.load_weights(os.path.join(cfg.path2, cfg.model, f'fold{i+1}.h5'))\n        model_all.append(model)\n    if cfg.plat=='local':\n        data_test = pd.read_parquet(os.path.join(cfg.path, 'train_series.parquet'))\n    elif cfg.plat=='kaggle':\n        data_test = pd.read_parquet(os.path.join(cfg.path, 'test_series.parquet'))\n    series = np.unique(data_test.series_id)\n    \n    sample_submission = pd.DataFrame()\n    \n    for i, serie in enumerate(series):\n        print(f'{serie} {i+1}/{len(series)}')\n        data_temp = data_test[data_test.series_id==serie].reset_index(drop=True)\n        data_temp['timestamp'] = data_temp.timestamp.str.split('T', expand=True).loc[:, 1].str.split(':', expand=True).loc[:, 0]\n        data_temp['timestamp'] = data_temp['timestamp'].astype(int)\n        index = np.arange(0, max(data_temp.step)-cfg.w_l, cfg.w_t)\n        data_save = np.zeros((max(1, len(index)), cfg.w_l, len(cfg.index)))\n        for j in range(len(index)):\n            data_save[j, :, 0] = data_temp.anglez[index[j]:(index[j] + cfg.w_l)]\n            data_save[j, :, 1] = data_temp.enmo[index[j]:(index[j] + cfg.w_l)]\n            data_save[j, :, 2] = data_temp.timestamp[index[j]:(index[j] + cfg.w_l)]\n        del data_temp\n        if cfg.normalization:\n            data_save = (data_save - np.mean(data_save, axis=1, keepdims=True)) / (np.std(data_save, axis=1, keepdims=True) + 1e-6)\n        else:\n            data_save[:, :, 0] = data_save[:, :, 0] / cfg.anglezmax\n            data_save[:, :, 2] = data_save[:, :, 2] / 24\n        for j in range(len(model_all)):\n            if j==0:\n                result = model_all[j].predict(data_save)[0]\n            else:\n                result = result + model_all[j].predict(data_save)[0]\n        result = result / len(model_all)\n        if len(index):\n            data_save = np.zeros(index[-1] + cfg.w_l)\n            data_temp = np.zeros(index[-1] + cfg.w_l)\n            for j in range(result.shape[0]):\n                data_save[j*cfg.w_t:(j*cfg.w_t + cfg.w_l)] = data_save[j*cfg.w_t:(j*cfg.w_t + cfg.w_l)] + result[j, :]\n                data_temp[j*cfg.w_t:(j*cfg.w_t + cfg.w_l)] = data_temp[j*cfg.w_t:(j*cfg.w_t + cfg.w_l)] + 1\n        else:\n            data_save = result[0, :]\n            data_temp = result[0, :] * 0 + 1\n        data_save = data_save / data_temp\n        \n        # data_save1 = scipy.signal.savgol_filter(data_save, cfg.smoothing_length, 3)\n        data_save2 = (data_save >= 0.5) + 0\n        data_save2 = data_save2[1:] - data_save2[:-1]\n        onset = np.where(data_save2==1)[0]\n        wakeup = np.where(data_save2==-1)[0]\n        if len(onset)>1:\n            if len(onset) > len(wakeup):\n                onset = onset[:-1]\n        if len(wakeup)>1:\n            if len(wakeup) > len(onset):\n                wakeup = wakeup[1:]\n        if len(onset)>1:\n            onset_bool = np.zeros(len(onset))\n            wakeup_bool = np.zeros(len(wakeup))\n            for j in range(len(onset)):\n                if np.sum(data_save[onset[j]:wakeup[j]] >= 0.5) >= ((wakeup[j] - onset[j]) * 0.8):\n                    onset_bool[j] = 1\n                    wakeup_bool[j] = 1\n            onset = onset[onset_bool==1]\n            wakeup = wakeup[wakeup_bool==1]\n        \n        while len(onset)>1 and sum((onset[1:] - wakeup[:-1]) < cfg.closing_size):\n            onset_bool = np.ones(len(onset))\n            wakeup_bool = np.ones(len(wakeup))\n            for j in range(1, len(onset)):\n                if (onset[j] - wakeup[j-1]) < cfg.closing_size:\n                    onset_bool[j] = 0\n                    wakeup_bool[j-1] = 0\n            onset = onset[onset_bool==1]\n            wakeup = wakeup[wakeup_bool==1]\n        \n        if len(onset)>1:\n            select = (wakeup - onset) >= cfg.closing_size\n            onset = onset[select]\n            wakeup = wakeup[select]\n\n        if len(onset):\n            step = np.concatenate((onset, wakeup))\n            event = np.concatenate((len(onset)*['onset'], len(wakeup)*['wakeup']))\n            df = pd.DataFrame({'series_id': serie, 'step': step, 'event': event, 'score': 1})\n            sample_submission = pd.concat((sample_submission, df))\n\n    sample_submission = sample_submission.reset_index(drop=True).reset_index(names='row_id')\n    sample_submission.to_csv('submission.csv', index=False)\n\n\n# %% main\nif __name__ == '__main__':\n    # 1. preprocessdata   (train on the local platform)\n    # preprocessdata()\n    \n    # 2. getinfo   (train on the local platform)\n    # dirs = getinfo(True)\n    \n    # 3. train   (train on the local platform)\n    # train(dirs)\n    \n    # 4. test\n    test()\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]}]}