{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Finding Critical Points using RNNs\n\nI will not be making the model weights public,\nmight release a training notebook later. \n\nRevealing the power of Neural Networks!\n\nCredits:\n* idea: https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/441470\n* dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi\n* arch: https://www.kaggle.com/competitions/tlvmc-parkinsons-freezing-gait-prediction/discussion/416410","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm \n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes\ntorch.set_num_interop_threads(4)\ntorch.set_num_threads(4)\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-15T02:54:01.876500Z","iopub.execute_input":"2023-10-15T02:54:01.877576Z","iopub.status.idle":"2023-10-15T02:54:09.253025Z","shell.execute_reply.started":"2023-10-15T02:54:01.877527Z","shell.execute_reply":"2023-10-15T02:54:09.251897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:54:17.453621Z","iopub.execute_input":"2023-10-15T02:54:17.454001Z","iopub.status.idle":"2023-10-15T02:54:17.459195Z","shell.execute_reply.started":"2023-10-15T02:54:17.453970Z","shell.execute_reply":"2023-10-15T02:54:17.458149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEMO_MODE = False\n    VERBOSE = True\n    \n    BATCH_SIZE = 32\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:54:17.941699Z","iopub.execute_input":"2023-10-15T02:54:17.942486Z","iopub.status.idle":"2023-10-15T02:54:17.947571Z","shell.execute_reply.started":"2023-10-15T02:54:17.942447Z","shell.execute_reply":"2023-10-15T02:54:17.946500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CMIDataReader:\n    def __init__(self, demo_mode, verbose):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n        self.verbose = verbose\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        if self.verbose:\n            print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        if self.verbose:\n            print(\"Percentage of removed rows : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n        data = data.bfill()\n        return data\n    \n    def reduce_memory_usage(self, data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        if self.verbose:\n            print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        if self.verbose:\n            print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n            print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_rows = next(pf.iter_batches(batch_size=100_000)) \n                data = pa.Table.from_batches([demo_rows]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nrows=100_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n        \n        if data_props[\"has_timestamp\"]:\n            data = self.cleaning(data)\n            data[\"timestamp\"] = pd.to_datetime(data[\"timestamp\"])\n            data = data.set_index(\"timestamp\")\n        \n        data = self.reduce_memory_usage(data)\n        gc.collect()\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:54:20.060569Z","iopub.execute_input":"2023-10-15T02:54:20.060925Z","iopub.status.idle":"2023-10-15T02:54:20.079350Z","shell.execute_reply.started":"2023-10-15T02:54:20.060895Z","shell.execute_reply":"2023-10-15T02:54:20.078218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = CMIDataReader(demo_mode=CFG.DEMO_MODE, verbose=CFG.VERBOSE)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:54:21.674201Z","iopub.execute_input":"2023-10-15T02:54:21.674574Z","iopub.status.idle":"2023-10-15T02:54:21.679428Z","shell.execute_reply.started":"2023-10-15T02:54:21.674544Z","shell.execute_reply":"2023-10-15T02:54:21.678403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"=\"*25)\nprint(\"LOAD SUBMISSION\")\nsubmission = reader.load_data(\"submission\")\nprint(submission.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"LOAD TEST SERIES\")\ntest_series = reader.load_data(\"test_series\")\nprint(test_series.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"=\"*25)\nprint(\"LOAD TRAIN SERIES\")\ntrain_series = reader.load_data(\"train_series\")\nprint(train_series.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"=\"*25)\nprint(\"LOAD TRAIN EVENTS\")\ntrain_events = reader.load_data(\"train_events\")\nprint(train_events.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:54:23.055083Z","iopub.execute_input":"2023-10-15T02:54:23.056218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBiGRU(nn.Module):\n    def __init__(self, hidden_size, n_layers=1, bidir=True):\n        super(ResidualBiGRU, self).__init__()\n\n        self.hidden_size = hidden_size\n        self.n_layers = n_layers\n\n        self.gru = nn.GRU(\n            hidden_size,\n            hidden_size,\n            n_layers,\n            batch_first=True,\n            bidirectional=bidir,\n        )\n        dir_factor = 2 if bidir else 1\n        self.fc1 = nn.Linear(\n            hidden_size * dir_factor, hidden_size * dir_factor * 2\n        )\n        self.ln1 = nn.LayerNorm(hidden_size * dir_factor * 2)\n        self.fc2 = nn.Linear(hidden_size * dir_factor * 2, hidden_size)\n        self.ln2 = nn.LayerNorm(hidden_size)\n\n    def forward(self, x, h=None):\n        res, new_h = self.gru(x, h)\n        # res.shape = (batch_size, sequence_size, 2*hidden_size)\n\n        res = self.fc1(res)\n        res = self.ln1(res)\n        res = nn.functional.relu(res)\n\n        res = self.fc2(res)\n        res = self.ln2(res)\n        res = nn.functional.relu(res)\n\n        # skip connection\n        res = res + x\n\n        return res, new_h\n\nclass MultiResidualBiGRU(nn.Module):\n    def __init__(self, input_size, hidden_size, out_size, n_layers, bidir=True):\n        super(MultiResidualBiGRU, self).__init__()\n\n        self.input_size = input_size\n        self.hidden_size = hidden_size\n        self.out_size = out_size\n        self.n_layers = n_layers\n\n        self.fc_in = nn.Linear(input_size, hidden_size)\n        self.ln = nn.LayerNorm(hidden_size)\n        self.res_bigrus = nn.ModuleList(\n            [\n                ResidualBiGRU(hidden_size, n_layers=1, bidir=bidir)\n                for _ in range(n_layers)\n            ]\n        )\n        self.fc_out = nn.Linear(hidden_size, out_size)\n\n    def forward(self, x, h=None):\n        # if we are at the beginning of a sequence (no hidden state)\n        if h is None:\n            # (re)initialize the hidden state\n            h = [None for _ in range(self.n_layers)]\n\n        x = self.fc_in(x)\n        x = self.ln(x)\n        x = nn.functional.relu(x)\n\n        new_h = []\n        for i, res_bigru in enumerate(self.res_bigrus):\n            x, new_hi = res_bigru(x, h[i])\n            new_h.append(new_hi)\n\n        x = self.fc_out(x)\n#         x = F.normalize(x,dim=0)\n        return x, new_h  # log probabilities + hidden states","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:21:05.057061Z","iopub.execute_input":"2023-10-15T02:21:05.057764Z","iopub.status.idle":"2023-10-15T02:21:05.068082Z","shell.execute_reply.started":"2023-10-15T02:21:05.057732Z","shell.execute_reply":"2023-10-15T02:21:05.067159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDataset(Dataset):\n    def __init__(\n        self,\n        series_ids,\n        series,\n    ):\n        series_ids = series_ids\n        series = series.reset_index()\n        self.data = []\n        \n        for viz_id in tqdm(series_ids):\n            self.data.append(series.loc[(series.series_id==viz_id)].copy().reset_index())\n            \n    def downsample_seq_generate_features(self, feat, downsample_factor):\n        \n        # 나눴을 때 12로 안 나눠  떨어지면 zero 패딩을 덧댐\n        if len(feat) % 12 != 0:\n            feat = np.concatenate([feat, np.zeros(12-((len(feat))%12))+feat[-1]])\n        feat = np.reshape(feat, (-1,12))\n        feat_mean = np.mean(feat,1)\n        feat_std = np.std(feat,1)\n        feat_median = np.median(feat,1)\n        feat_max = np.max(feat,1)\n        feat_min = np.min(feat,1)\n\n        return np.dstack([feat_mean,feat_std,feat_median,feat_max,feat_min])[0]\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, index):\n        X = self.data[index][['anglez','enmo']].values.astype(np.float32)\n        X = np.concatenate([self.downsample_seq_generate_features(X[:,i],12) for i in range(X.shape[1])],-1)\n        X = torch.from_numpy(X)\n        return X\n\ntest_ds = SleepDataset(test_series.series_id.unique(),test_series)\ndel test_series\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:04:30.935204Z","iopub.execute_input":"2023-10-15T02:04:30.935554Z","iopub.status.idle":"2023-10-15T02:04:31.138751Z","shell.execute_reply.started":"2023-10-15T02:04:30.935529Z","shell.execute_reply":"2023-10-15T02:04:31.137787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_chunk_size = 24*60*100\nmin_interval = 30","metadata":{"execution":{"iopub.status.busy":"2023-10-15T01:45:27.060077Z","iopub.execute_input":"2023-10-15T01:45:27.060758Z","iopub.status.idle":"2023-10-15T01:45:27.065177Z","shell.execute_reply.started":"2023-10-15T01:45:27.060728Z","shell.execute_reply":"2023-10-15T01:45:27.064153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train code 작성\n## custom dataset을 다시 작성 ","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## load train data","metadata":{}},{"cell_type":"code","source":"train_series = reader.load_data(data_name=\"train_series\")\nids = train_series.series_id.unique()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:53:57.178897Z","iopub.execute_input":"2023-10-15T02:53:57.179333Z","iopub.status.idle":"2023-10-15T02:53:57.410463Z","shell.execute_reply.started":"2023-10-15T02:53:57.179297Z","shell.execute_reply":"2023-10-15T02:53:57.409148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:45:13.885430Z","iopub.execute_input":"2023-10-15T02:45:13.885799Z","iopub.status.idle":"2023-10-15T02:45:13.892414Z","shell.execute_reply.started":"2023-10-15T02:45:13.885769Z","shell.execute_reply":"2023-10-15T02:45:13.891407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = SleepDataset(train_series.series_id.unique(),train_series)\ndel train_series\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:44:40.359808Z","iopub.execute_input":"2023-10-15T02:44:40.360196Z","iopub.status.idle":"2023-10-15T02:44:40.386355Z","shell.execute_reply.started":"2023-10-15T02:44:40.360167Z","shell.execute_reply":"2023-10-15T02:44:40.385077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train mode\nmodel = MultiResidualBiGRU(input_size=10, hidden_size=64, out_size=2, n_layers=5).to(device).train()\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T02:21:13.791322Z","iopub.execute_input":"2023-10-15T02:21:13.791670Z","iopub.status.idle":"2023-10-15T02:21:18.941886Z","shell.execute_reply.started":"2023-10-15T02:21:13.791644Z","shell.execute_reply":"2023-10-15T02:21:18.940816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(train_ids)):\n    X = test_ds[i].half()\n    \n    seq_len = X.shape[0]\n    h = None\n    pred = torch.zeros((len(X),2)).half()\n    for j in range(0, seq_len, max_chunk_size):\n        y_pred, h = model(X[j: j + max_chunk_size].float(), h)\n        h = [hi.detach() for hi in h]\n        pred[j : j + max_chunk_size] = y_pred.detach()\n        del y_pred;gc.collect()\n    del h,X;gc.collect()\n    pred = pred.numpy()\n    \n    series_id = ids[i]\n    \n    days = len(pred)/(17280/12)\n    scores0, scores1 = np.zeros(len(pred),dtype=np.float16),np.zeros(len(pred),dtype=np.float16)\n    for index in range(len(pred)):\n        if pred[index,0]==max(pred[max(0,index-min_interval):index+min_interval,0]):\n            scores0[index] = max(pred[max(0,index-min_interval):index+min_interval,0])\n        if pred[index,1]==max(pred[max(0,index-min_interval):index+min_interval,1]):\n            scores1[index] = max(pred[max(0,index-min_interval):index+min_interval,1])\n    candidates_onset = np.argsort(scores0)[-max(1,round(days)):]\n    candidates_wakeup = np.argsort(scores1)[-max(1,round(days)):]\n    \n    onset = test_ds.data[i][['step']].iloc[np.clip(candidates_onset*12,0,len(test_ds.data[i])-1)].astype(np.int32)\n    onset['event'] = 'onset'\n    onset['series_id'] = series_id\n    onset['score']= scores0[candidates_onset]\n    wakeup = test_ds.data[i][['step']].iloc[np.clip(candidates_wakeup*12,0,len(test_ds.data[i])-1)].astype(np.int32)\n    wakeup['event'] = 'wakeup'\n    wakeup['series_id'] = series_id\n    wakeup['score']= scores1[candidates_wakeup]\n    submission = pd.concat([submission,onset,wakeup],axis=0)\n    del onset,wakeup,candidates_onset,candidates_wakeup,scores0,scores1,pred,series_id,\n    gc.collect()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MultiResidualBiGRU(input_size=10, hidden_size=64, out_size=2, n_layers=5).to(device).eval()\nmodel.load_state_dict(torch.load(f'/kaggle/input/sleep-point-models/model_0.pth',map_location=device))\nsubmission = pd.DataFrame()\n\nfor i in range(len(test_ds)):\n    X = test_ds[i].half()\n    \n    seq_len = X.shape[0]\n    h = None\n    pred = torch.zeros((len(X),2)).half()\n    for j in range(0, seq_len, max_chunk_size):\n        y_pred, h = model(X[j: j + max_chunk_size].float(), h)\n        h = [hi.detach() for hi in h]\n        pred[j : j + max_chunk_size] = y_pred.detach()\n        del y_pred;gc.collect()\n    del h,X;gc.collect()\n    pred = pred.numpy()\n    \n    series_id = ids[i]\n    \n    days = len(pred)/(17280/12)\n    scores0, scores1 = np.zeros(len(pred),dtype=np.float16),np.zeros(len(pred),dtype=np.float16)\n    for index in range(len(pred)):\n        if pred[index,0]==max(pred[max(0,index-min_interval):index+min_interval,0]):\n            scores0[index] = max(pred[max(0,index-min_interval):index+min_interval,0])\n        if pred[index,1]==max(pred[max(0,index-min_interval):index+min_interval,1]):\n            scores1[index] = max(pred[max(0,index-min_interval):index+min_interval,1])\n    candidates_onset = np.argsort(scores0)[-max(1,round(days)):]\n    candidates_wakeup = np.argsort(scores1)[-max(1,round(days)):]\n    \n    onset = test_ds.data[i][['step']].iloc[np.clip(candidates_onset*12,0,len(test_ds.data[i])-1)].astype(np.int32)\n    onset['event'] = 'onset'\n    onset['series_id'] = series_id\n    onset['score']= scores0[candidates_onset]\n    wakeup = test_ds.data[i][['step']].iloc[np.clip(candidates_wakeup*12,0,len(test_ds.data[i])-1)].astype(np.int32)\n    wakeup['event'] = 'wakeup'\n    wakeup['series_id'] = series_id\n    wakeup['score']= scores1[candidates_wakeup]\n    submission = pd.concat([submission,onset,wakeup],axis=0)\n    del onset,wakeup,candidates_onset,candidates_wakeup,scores0,scores1,pred,series_id,\n    gc.collect()\nsubmission = submission.sort_values(['series_id','step']).reset_index(drop=True)\nsubmission['row_id'] = submission.index.astype(int)\nsubmission['score'] = submission['score'].fillna(submission['score'].mean())\nsubmission = submission[['row_id','series_id','step','event','score']]\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T01:46:17.383789Z","iopub.execute_input":"2023-10-15T01:46:17.384142Z","iopub.status.idle":"2023-10-15T01:46:22.982357Z","shell.execute_reply.started":"2023-10-15T01:46:17.384106Z","shell.execute_reply":"2023-10-15T01:46:22.981103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}