{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6507308,"sourceType":"datasetVersion","datasetId":3706667}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Introduction</span>**\n\nSleep is crucial in regulating mood, emotions, and behavior in individuals of all ages, particularly children. Understanding how environmental factors impact sleep, mood, and behavior can inform the development of personalized interventions and support systems tailored to the unique needs of each child. By accurately detecting periods of sleep and wakefulness from wrist-worn accelerometer data, researchers can gain a deeper understanding of sleep patterns and better understand disturbances in children.\n\n# **<span style=\"color:#F7B2B0;\">Goal</span>**\n\nThe goal of this competition is to detect sleep onset and wake by developing a model trained on wrist-worn accelerometer data in order to determine a person's sleep state.\n","metadata":{}},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Install the Libraries</span>**","metadata":{}},{"cell_type":"code","source":"!pip install -q wandb\n!pip install -q tensorflow_addons","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:11:39.090647Z","iopub.execute_input":"2023-10-23T13:11:39.091137Z","iopub.status.idle":"2023-10-23T13:11:52.063587Z","shell.execute_reply.started":"2023-10-23T13:11:39.091103Z","shell.execute_reply":"2023-10-23T13:11:52.062464Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Import the Packages</span>**","metadata":{}},{"cell_type":"code","source":"\nfrom itertools import groupby\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nimport tensorflow.keras as keras\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom kaggle_secrets import UserSecretsClient\nimport wandb\nfrom datetime import datetime\n\nimport tensorflow as tf\nimport random\nfrom copy import deepcopy\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import Input, Dropout, Dense, ReLU, BatchNormalization, Activation, Concatenate\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow_addons.activations import sparsemax\nimport tensorflow as tf\nimport random\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import Input, Dropout, Dense, ReLU, BatchNormalization, Activation, Concatenate\nfrom copy import deepcopy","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:11:53.159325Z","iopub.execute_input":"2023-10-23T13:11:53.159601Z","iopub.status.idle":"2023-10-23T13:11:54.156931Z","shell.execute_reply.started":"2023-10-23T13:11:53.159576Z","shell.execute_reply":"2023-10-23T13:11:54.155902Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">Data Pipeline</span>**","metadata":{}},{"cell_type":"code","source":"train = pd.read_parquet(\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\")\ntest  = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n\n# parse the timestamp and create an \"hour\" feature\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"],utc=True)\ntrain[\"hour\"] = train[\"timestamp\"].dt.hour\n\ntest[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"],utc=True)\ntest[\"hour\"] = test[\"timestamp\"].dt.hour","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:11:54.159039Z","iopub.execute_input":"2023-10-23T13:11:54.161113Z","iopub.status.idle":"2023-10-23T13:13:36.875356Z","shell.execute_reply.started":"2023-10-23T13:11:54.161083Z","shell.execute_reply":"2023-10-23T13:13:36.874338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = train[[\"anglez\",\"enmo\",\"hour\"]]\nytrain = train[\"awake\"]\n\nX_train, X_test, y_train, y_test = train_test_split(x_train, ytrain, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:13:36.877011Z","iopub.execute_input":"2023-10-23T13:13:36.877292Z","iopub.status.idle":"2023-10-23T13:13:40.151895Z","shell.execute_reply.started":"2023-10-23T13:13:36.877268Z","shell.execute_reply":"2023-10-23T13:13:40.150793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://camo.githubusercontent.com/dd842f7b0be57140e68b2ab9cb007992acd131c48284eaf6b1aca758bfea358b/68747470733a2f2f692e696d6775722e636f6d2f52557469567a482e706e67\">\n\n> I will be integrating W&B for visualizations and logging artifacts!\n> \n> [Sleep State Detection Project on W&B Dashboard](https://wandb.ai/usharengaraju/child_mind)\n> \n> - To get the API key, create an account in the [website](https://wandb.ai/site) .\n> - Use secrets to use API Keys more securely ","metadata":{}},{"cell_type":"code","source":"# Setup user secrets for login\nuser_secrets = UserSecretsClient()\nwandb_api = user_secrets.get_secret(\"api_key\") \nwandb.login(key=wandb_api)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:13:40.153407Z","iopub.execute_input":"2023-10-23T13:13:40.153811Z","iopub.status.idle":"2023-10-23T13:13:41.401323Z","shell.execute_reply.started":"2023-10-23T13:13:40.153761Z","shell.execute_reply":"2023-10-23T13:13:41.400313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#run = wandb.init(project = 'child_mind',\n#                 config = {},\n#                 save_code = True,\n                 \n#)\n#table = wandb.Table(dataframe=train[[\"anglez\",\"enmo\",\"hour\",\"awake\"]])\n#wandb.log({\"Table\":table})\n#run.finish()\n","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:13:41.402589Z","iopub.execute_input":"2023-10-23T13:13:41.402962Z","iopub.status.idle":"2023-10-23T13:13:42.265143Z","shell.execute_reply.started":"2023-10-23T13:13:41.402910Z","shell.execute_reply":"2023-10-23T13:13:42.264162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">GrowNet</span>**","metadata":{}},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\nrun = wandb.init(project = 'child_mind',\n                 save_code = True,\n                 name='grownet'\n                 \n)","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:13:43.092356Z","iopub.execute_input":"2023-10-23T13:13:43.092646Z","iopub.status.idle":"2023-10-23T13:14:23.395797Z","shell.execute_reply.started":"2023-10-23T13:13:43.092619Z","shell.execute_reply":"2023-10-23T13:14:23.394962Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:14:23.398864Z","iopub.execute_input":"2023-10-23T13:14:23.399209Z","iopub.status.idle":"2023-10-23T13:14:23.405394Z","shell.execute_reply.started":"2023-10-23T13:14:23.399179Z","shell.execute_reply":"2023-10-23T13:14:23.404581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass DynamicNet(object):\n    def __init__(self, c0 = None, lr = None, concat_input=False, additive_boosting=False, encoder_layers=None):\n        self.models = []\n        self.c0 = tf.Variable(np.float32(c0) if c0 is not None else random.uniform(0.0, 1.0))\n        self.lr = lr\n        self.boost_rate  = tf.Variable(lr if lr is not None else random.uniform(0.0, 1.0))\n        self.concat_input = None\n        self.additive_boosting = False\n        self.encoder_layers = encoder_layers\n    def freeze_all_networks(self):\n        for model in self.models:\n            for l in model.layers: l.trainable = False\n    def unfreeze_all_networks(self):\n        for model in self.models:\n            for l in model.layers: l.trainable = True\n    def add(self, model):\n        last_activation = 'sigmoid'\n        if last_activation in ['sigmoid', 'softmax']: model.layers[-1].activation = None\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.loss = model.loss\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.optimizer = model.optimizer\n        if hasattr(model, 'optimizer') and model.optimizer is not None: self.lr = model.optimizer.lr if self.lr is None else self.lr\n        if len(self.models) == 0:\n            self.models = [model]\n            self.full_model = self.models[-1]\n            self.embed_full_model = self.models[-1]\n        else: self.models.append(model)\n        full_inp = Input(shape=3)\n        out_orig = self.embed_full_model(full_inp)\n        out = out_orig\n        if self.concat_input: out = Concatenate()([out, full_inp])\n        if len(self.models) > 1:\n            # print(out)\n            # print(K.int_shape(out))\n            if K.int_shape(out) != K.int_shape(self.models[-2].input): out = Dense(K.int_shape(self.models[-1].input)[-1])(out)\n            new_out = self.models[-1](out)\n        else: new_out = self.models[-1](full_inp)\n        new_full_out = (self.c0) + (self.boost_rate * new_out)\n        self.full_model = Model(full_inp, Activation(last_activation)(new_full_out))\n        if self.encoder_layers is not None:\n            if len(self.models) > 1: self.embed_full_model = Model(full_inp, Model(self.models[-1].input, self.models[-1].layers[self.encoder_layers].output)(out))\n            else: \n              # print(self.models)\n              # print(self.models[-1].input)\n              # print(self.models[-1].layers[self.encoder_layers])\n              self.embed_full_model = Model(full_inp, Model(self.models[-1].input, self.models[-1].layers[self.encoder_layers].output)(full_inp))\n        else: self.embed_full_model = Model(full_inp, new_out)\n\n    def fit(self, x_train, y_train, lr, w_decay=0.0, epochs=10, validation_data = None, **kwargs):\n        if self.optimizer is None: optimizer = Adam(lr, decay=w_decay)\n        else: optimizer =self.optimizer\n        self.full_model.compile(optimizer, self.loss,['binary_accuracy'])\n        self.full_model.fit(x_train, y_train, epochs=epochs, validation_data = (x_train, y_train), **kwargs)\n    def predict(self, x_train, **kwargs): return self.full_model.predict(x_train, **kwargs)\n\n\nclass GradientBoost(object):\n    def __init__(self, base_model, lr = 1e-3, weight_decay = 1e-5, early_stopping_steps = 5, batch_size = 256, correct_epoch = 1, model_order = \"second\", n_boosting_rounds = 5 , boost_rate = 1.0, hidden_size=512, epochs_per_stage=5, encoder_layers=2):\n        self.lr = lr\n        self.base_model = base_model\n        self.batch_size = batch_size\n        self.boost_rate = boost_rate\n        self.model_order = model_order\n        self.hidden_size = hidden_size\n        self.weight_decay = weight_decay\n        self.num_nets = n_boosting_rounds\n        self.encoder_layers = encoder_layers\n        self.correct_epoch = correct_epoch\n        self.epochs_per_stage = epochs_per_stage\n        self.early_stopping_steps = early_stopping_steps\n        self.base_model.summary()\n\n    def fit(self, x_train, y_train, validation_data = None, n_boosting_rounds=None, correct_epoch=None, epochs_per_stage=None, **kwargs):\n        self.num_nets = n_boosting_rounds if n_boosting_rounds is not None else self.num_nets\n        self.correct_epoch = correct_epoch if correct_epoch is not None else self.correct_epoch\n        self.epochs_per_stage = epochs_per_stage if epochs_per_stage is not None else self.epochs_per_stage\n        x_val , y_val = validation_data if validation_data is not None else (None, None)\n        net_ensemble = DynamicNet(concat_input=True, encoder_layers=self.encoder_layers)\n        lr = self.lr\n        L2 = self.weight_decay\n        for stage in range(self.num_nets):\n            print(\"Stage is {}\".format(stage))\n            params = {}\n            params[\"feat_d\"] = x_train.shape[1]\n            params[\"hidden_size\"] = self.hidden_size\n            new_model = self.base_model\n            new_model.optimizer = self.base_model.optimizer\n            new_model.loss = self.base_model.loss\n            net_ensemble.freeze_all_networks()\n            net_ensemble.add(new_model)\n            net_ensemble.fit(x_train, y_train,self.lr, epochs=self.epochs_per_stage,batch_size=self.batch_size,  validation_data = validation_data,callbacks=[WandbCallback()])\n            lr_scaler = 2\n            if stage != 0:\n                if stage % 3 == 0: lr /= 2\n                net_ensemble.unfreeze_all_networks()\n                net_ensemble.fit(x_train, y_train,lr / lr_scaler, epochs=self.correct_epoch,batch_size=self.batch_size,  w_decay=L2, validation_data = validation_data,callbacks=[WandbCallback()])\n        self.model = net_ensemble\n\n    def predict(self, x_test, **kwargs): return self.model.predict(x_test, **kwargs)\n\n\ninp = Input(3)\nl = BatchNormalization()(inp)\nl = ReLU()(l)\nl = Dense(32)(l)\nl = BatchNormalization()(l)\nl = ReLU()(l)\nl = Dropout(0.4)(l)\nl = Dense(32)(l)\nl = BatchNormalization()(l)\nl = ReLU()(l)\nout = Dense(1, activation='sigmoid')(l)\nsimple_model = Model(inp, out)\nsimple_model.compile(Adam(0.001), 'binary_crossentropy',metrics=['binary_accuracy'])\nbs = 4096*2\nmodel = GradientBoost(simple_model,batch_size=bs)\nmodel.fit(X_train, y_train,epochs=10,batch_size=bs,validation_data=(X_test,y_test))","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:14:23.406418Z","iopub.execute_input":"2023-10-23T13:14:23.407091Z","iopub.status.idle":"2023-10-23T13:19:29.387581Z","shell.execute_reply.started":"2023-10-23T13:14:23.407056Z","shell.execute_reply":"2023-10-23T13:19:29.386631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{"execution":{"iopub.status.busy":"2023-10-23T13:19:29.389207Z","iopub.execute_input":"2023-10-23T13:19:29.389880Z","iopub.status.idle":"2023-10-23T13:19:34.445564Z","shell.execute_reply.started":"2023-10-23T13:19:29.389836Z","shell.execute_reply":"2023-10-23T13:19:34.444613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">TabNet</span>**","metadata":{}},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\nrun = wandb.init(project = 'child_mind',\n                 save_code = True,\n                 name='tabnet'\n                 \n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef GLU(x):\n    return x * tf.sigmoid(x)\n\nclass FCBlock(layers.Layer):\n    def __init__(self, units):\n        super().__init__()\n        self.layer = layers.Dense(units)\n        self.bn = layers.BatchNormalization()\n\n    def call(self, x):\n        return GLU(self.bn(self.layer(x)))\n\nclass SharedBlock(layers.Layer):\n    def __init__(self, units, mult=tf.sqrt(0.5)):\n        super().__init__()\n        self.layer1 = FCBlock(units)\n        self.layer2 = FCBlock(units)\n        self.mult = mult\n\n    def call(self, x):\n        out1 = self.layer1(x)\n        out2 = self.layer2(out1)\n        return out2 + self.mult * out1\n\nclass DecisionBlock(SharedBlock):\n    def __init__(self, units, mult=tf.sqrt(0.5)):\n        super().__init__(units, mult)\n\n    def call(self, x):\n        out1 = x * self.mult + self.layer1(x)\n        out2 = out1 * self.mult + self.layer2(out1)\n        return out2\n\nclass Prior(layers.Layer):\n    def __init__(self, gamma=1.1):\n        super().__init__()\n        self.gamma = gamma\n\n    def reset(self):\n        self.P = 1.0\n\n    def call(self, mask):\n        self.P = self.P * (self.gamma - mask)\n        return self.P\nclass AttentiveTransformer(layers.Layer):\n    def __init__(self, units):\n        super().__init__()\n        self.layer = layers.Dense(units)\n        self.bn = layers.BatchNormalization()\n\n    def call(self, x, prior):\n        return sparsemax(prior * self.bn(self.layer(x)))\nclass TabNet(keras.Model):\n    def __init__(self, input_dim, output_dim, steps, n_d, n_a, gamma=1.3):\n        super().__init__()\n        # hyper-parameters\n        self.n_d, self.n_a, self.steps = n_d, n_a, steps\n        # input-normalisation\n        self.bn = layers.BatchNormalization()\n        # Feature Transformer\n        self.shared = SharedBlock(n_d+n_a)\n        self.first_block = DecisionBlock(n_d+n_a)\n        self.decision_blocks = [DecisionBlock(n_d+n_a)] * steps\n        # Attentive Transformer\n        self.attention = [AttentiveTransformer(input_dim)] * steps\n        self.prior_scale = Prior(gamma)\n        # final layer\n        self.final = layers.Dense(output_dim,activation='relu')\n\n        self.eps = 1e-8\n        self.add_layer = layers.Add()\n\n    @tf.function\n    def call(self, x):\n        self.prior_scale.reset()\n        final_outs = []\n        mask_losses = []\n\n        x = self.bn(x)\n        attention = self.first_block(self.shared(x))[:,:self.n_a]\n        for i in range(self.steps):\n            mask = self.attention[i](attention, self.prior_scale.P)\n            entropy = mask * tf.math.log(mask + self.eps)\n            mask_losses.append(\n                -tf.reduce_sum(entropy, axis=-1) / self.steps\n            )\n\n            prior = self.prior_scale(mask)\n            out = self.decision_blocks[i](self.shared(x * prior))\n            attention, output = out[:,:self.n_a], out[:,self.n_a:]\n            final_outs.append(tf.nn.relu(output))\n\n        final_out = self.add_layer(final_outs)\n        mask_loss = self.add_layer(mask_losses)\n\n        return self.final(final_out)\n\n    def mask_importance(self, x):\n        self.prior_scale.reset()\n        feature_importance = 0\n\n        x = self.bn(x)\n        attention = self.first_block(self.shared(x))[:,:self.n_a]\n        for i in range(self.steps):\n            mask = self.attention[i](attention, self.prior_scale.P)\n\n            prior = self.prior_scale(mask)\n            out = self.decision_blocks[i](self.shared(x * prior))\n            attention, output = out[:,:self.n_a], out[:,self.n_a:]\n            step_importance = tf.reduce_sum(tf.nn.relu(output), axis=1, keepdims=True)\n            feature_importance += mask * step_importance\n\n        return feature_importance","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basemodel = TabNet(128, 64, 2, 10, 10, 1.3)\n\ninp = Input(3)\nl = BatchNormalization()(inp)\nl = Dense(128,activation='relu')(l)\nl = basemodel(l)\nl = Activation('relu')(l)\nout = Dense(1,activation='sigmoid')(l)\n\nmodel = Model(inp, out)\nmodel.compile(Adam(0.001), 'binary_crossentropy',metrics=['binary_accuracy'])\nmodel.summary()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train, y_train,epochs=10,batch_size = 4096*2, validation_data=(X_test,y_test),callbacks=[WandbMetricsLogger()])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **<span style=\"color:#F7B2B0;\">GRUNN</span>**","metadata":{}},{"cell_type":"code","source":"from wandb.keras import WandbCallback, WandbMetricsLogger\n\nrun = wandb.init(project = 'child_mind',\n                 save_code = True,\n                 name='GRUNN'\n                 \n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport random\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, clone_model\nfrom tensorflow.keras.layers import *","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(3)\nl = ReLU()(inp)\nl = tf.expand_dims(l,axis=1)\nl = GRU(32, activation='relu',return_sequences=True, return_state=True)(l)\n\nl = GRU(32, activation='relu', return_sequences=True, return_state=True)(l)\nl = GRU(32, activation='relu')(l)\n# l=Flatten()(l)\nout = Dense(1, activation='sigmoid')(l)\nmodel = Model(inp, out)\nmodel.compile(Adam(0.001), 'binary_crossentropy',metrics=['binary_accuracy'])\nbs = 4096*2\nmodel.fit(X_train, y_train,epochs=10,batch_size=bs,validation_data=(X_test,y_test),callbacks=[WandbCallback()])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run.finish()","metadata":{},"execution_count":null,"outputs":[]}]}