{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Finding Critical Points using RNNs\n\n - Data Prep notebook - this\n - [Training notebook](https://www.kaggle.com/werus23/sleep-critical-point-train)\n - [Inference Notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-infer)\n\nCredits:\n - dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm \n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-24T14:59:59.853141Z","iopub.execute_input":"2023-11-24T14:59:59.853776Z","iopub.status.idle":"2023-11-24T15:00:10.788030Z","shell.execute_reply.started":"2023-11-24T14:59:59.853750Z","shell.execute_reply":"2023-11-24T15:00:10.787261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:03:38.512930Z","iopub.execute_input":"2023-11-24T15:03:38.513306Z","iopub.status.idle":"2023-11-24T15:03:38.518703Z","shell.execute_reply.started":"2023-11-24T15:03:38.513276Z","shell.execute_reply":"2023-11-24T15:03:38.517681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEMO_MODE = True","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:03:39.922962Z","iopub.execute_input":"2023-11-24T15:03:39.923325Z","iopub.status.idle":"2023-11-24T15:03:39.927683Z","shell.execute_reply.started":"2023-11-24T15:03:39.923295Z","shell.execute_reply":"2023-11-24T15:03:39.926756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed steps : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_steps = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_steps]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nsteps=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:03:40.778819Z","iopub.execute_input":"2023-11-24T15:03:40.779523Z","iopub.status.idle":"2023-11-24T15:03:40.801768Z","shell.execute_reply.started":"2023-11-24T15:03:40.779491Z","shell.execute_reply":"2023-11-24T15:03:40.800847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"train_series\")\nevents = reader.load_data(data_name=\"train_events\")","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:05:27.887005Z","iopub.execute_input":"2023-11-24T15:05:27.887749Z","iopub.status.idle":"2023-11-24T15:08:22.662708Z","shell.execute_reply.started":"2023-11-24T15:05:27.887714Z","shell.execute_reply":"2023-11-24T15:08:22.661774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\nids = series.series_id.unique()\n\nfor viz_id in tqdm(ids):\n    viz_targets = []\n    viz_events = events[events.series_id == viz_id]\n    viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n    viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n    viz_series['hour'] = viz_series['dt'].dt.hour\n\n    check = 0\n    for i in range(len(viz_events)-1):\n        if viz_events.iloc[i].event =='onset' and viz_events.iloc[i+1].event =='wakeup' and viz_events.iloc[i].night==viz_events.iloc[i+1].night:\n            start,end = viz_events.timestamp.iloc[i],viz_events.timestamp.iloc[i+1]\n\n            start_id = viz_series.loc[viz_series.timestamp ==start].index.values[0]\n            end_id = viz_series.loc[viz_series.timestamp ==end].index.values[0]\n            viz_targets.append((start_id,end_id))\n            check+=1\n    targets.append(viz_targets)\n    data.append(viz_series[['anglez','enmo','step']]) #you can include features like hour or minnute or second if you want to\n","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:08:22.664549Z","iopub.execute_input":"2023-11-24T15:08:22.665217Z","iopub.status.idle":"2023-11-24T15:46:38.571066Z","shell.execute_reply.started":"2023-11-24T15:08:22.665164Z","shell.execute_reply":"2023-11-24T15:46:38.570129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = joblib.dump((targets,data,ids), 'train_data.pkl')\ntrain_data","metadata":{"execution":{"iopub.status.busy":"2023-11-24T15:57:00.002188Z","iopub.execute_input":"2023-11-24T15:57:00.002589Z","iopub.status.idle":"2023-11-24T15:57:01.381513Z","shell.execute_reply.started":"2023-11-24T15:57:00.002559Z","shell.execute_reply":"2023-11-24T15:57:01.380530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SIGMA = 720 #average length of day is 24*60*12 = 17280 for comparison\n# SAMPLE_FREQ = 12 # 1 obs per minute\n# def normalize(y):\n#     mean = y[:,0].mean().item()\n#     std = y[:,0].std().item()\n#     y[:,0] = (y[:,0]-mean)/(std+1e-16)\n#     mean = y[:,1].mean().item()\n#     std = y[:,1].std().item()\n#     y[:,1] = (y[:,1]-mean)/(std+1e-16)\n#     return y\n\n# class SleepDataset(Dataset):\n#     def __init__(\n#         self,\n#         file\n#     ):\n#         self.targets,self.data,self.ids = joblib.load(file)\n            \n#     def downsample_seq_generate_features(self,feat, downsample_factor = SAMPLE_FREQ):\n#         # downsample data and generate features\n#         if len(feat)%SAMPLE_FREQ==0:\n#             feat = np.concatenate([feat,np.zeros(SAMPLE_FREQ-((len(feat))%SAMPLE_FREQ))+feat[-1]])\n#         feat = np.reshape(feat, (-1,SAMPLE_FREQ))\n#         feat_mean = np.mean(feat,1)\n#         feat_std = np.std(feat,1)\n#         feat_median = np.median(feat,1)\n#         feat_max = np.max(feat,1)\n#         feat_min = np.min(feat,1)\n\n#         return np.dstack([feat_mean,feat_std,feat_median,feat_max,feat_min])[0]\n#     def downsample_seq(self,feat, downsample_factor = SAMPLE_FREQ):\n#         # downsample data\n#         if len(feat)%SAMPLE_FREQ==0:\n#             feat = np.concatenate([feat,np.zeros(SAMPLE_FREQ-((len(feat))%SAMPLE_FREQ))+feat[-1]])\n#         feat = np.reshape(feat, (-1,SAMPLE_FREQ))\n#         feat_mean = np.mean(feat,1)\n#         return feat_mean\n    \n#     def gauss(self,n=SIGMA,sigma=SIGMA*0.15):\n#         # guassian distribution function\n#         r = range(-int(n/2),int(n/2)+1)\n#         return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]\n    \n#     def __len__(self):\n#         return len(self.targets)\n\n#     def __getitem__(self, index):\n#         X = self.data[index][['anglez','enmo']]\n#         y = self.targets[index]\n        \n#         # turn target inds into array\n#         target_guassian = np.zeros((len(X),2))\n#         for s,e in y:\n#             st1,st2 = max(0,s-SIGMA//2),s+SIGMA//2+1\n#             ed1,ed2 = e-SIGMA//2,min(len(X),e+SIGMA//2+1)\n#             target_guassian[st1:st2,0] = self.gauss()[st1-(s-SIGMA//2):]\n#             target_guassian[ed1:ed2,1] = self.gauss()[:SIGMA+1-((e+SIGMA//2+1)-ed2)]\n#             gc.collect()\n#         y = target_guassian\n#         gc.collect()\n#         X = np.concatenate([self.downsample_seq_generate_features(X.values[:,i],SAMPLE_FREQ) for i in range(X.shape[1])],-1)\n#         gc.collect()\n#         y = np.dstack([self.downsample_seq(y[:,i],SAMPLE_FREQ) for i in range(y.shape[1])])[0]\n#         gc.collect()\n#         y = normalize(torch.from_numpy(y))\n#         X = torch.from_numpy(X)\n#         return X, y\n# train_ds = SleepDataset('/kaggle/working/train_data.pkl')\n\n# # GPU 사용 가능 여부 확인\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:10:36.694530Z","iopub.status.idle":"2023-11-24T14:10:36.694848Z","shell.execute_reply.started":"2023-11-24T14:10:36.694689Z","shell.execute_reply":"2023-11-24T14:10:36.694704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch\n# from torch import nn\n# from torch.utils.data import Dataset, DataLoader, SubsetRandomSampler\n# import torch.optim as optim\n# from tqdm import tqdm\n# import joblib\n# import numpy as np\n# import gc\n\n\n# EPOCHS = 5\n# WARMUP_PROP = 0.2\n# BS = 1\n# WORKERS = 4\n# TRAIN_PROP = 0.8\n# max_chunk_size = 150000\n\n# class TransformerEncoderLayer(nn.Module):\n#     def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1):\n#         super(TransformerEncoderLayer, self).__init__()\n#         self.self_attn = nn.MultiheadAttention(d_model, nhead)\n#         self.linear1 = nn.Linear(d_model, dim_feedforward)\n#         self.dropout = nn.Dropout(dropout)\n#         self.linear2 = nn.Linear(dim_feedforward, d_model)\n#         self.norm1 = nn.LayerNorm(d_model)\n#         self.norm2 = nn.LayerNorm(d_model)\n#         self.dropout1 = nn.Dropout(dropout)\n#         self.dropout2 = nn.Dropout(dropout)\n\n#     def forward(self, src, src_mask=None):\n#         src2 = self.self_attn(src, src, src, attn_mask=src_mask)[0]\n#         src = src + self.dropout1(src2)\n#         src = self.norm1(src)\n\n#         src2 = self.linear2(self.dropout(torch.relu(self.linear1(src))))\n#         src = src + self.dropout2(src2)\n#         src = self.norm2(src)\n#         return src\n\n# class TransformerEncoder(nn.Module):\n#     def __init__(self, num_layers, input_size, hidden_size, out_size, nhead, dim_feedforward=2048, dropout=0.1):\n#         super(TransformerEncoder, self).__init__()\n#         self.input_size = input_size\n#         self.hidden_size = hidden_size\n#         self.out_size = out_size\n#         self.nhead = nhead\n#         self.num_layers = num_layers\n\n#         # Define the initial linear layer\n#         self.fc_in = nn.Linear(input_size, hidden_size)\n#         self.ln = nn.LayerNorm(hidden_size)\n\n#         # Create multiple layers of TransformerEncoderLayer\n#         self.transformer_encoder_layers = nn.ModuleList([\n#             TransformerEncoderLayer(hidden_size, nhead, dim_feedforward, dropout)\n#             for _ in range(num_layers)\n#         ])\n\n#         # Output layer\n#         self.fc_out = nn.Linear(hidden_size, out_size)\n\n#     def forward(self, x):\n# #         x = self.fc_in(x)\n#         x = self.fc_in(x.float())\n\n#         x = self.ln(x)\n        \n#         # Apply TransformerEncoderLayer multiple times\n#         for layer in self.transformer_encoder_layers:\n#             x = layer(x)\n        \n#         # Apply output layer\n#         x = self.fc_out(x)\n#         return x\n\n# # 수정된 모델 정의\n# num_layers = 6  # 수정: 레이어 수 조정 가능\n# input_size = 10  # 수정: 입력 차원에 맞게 조정\n# hidden_size = 64  # 수정: 은닉 상태 크기에 맞게 조정\n# out_size = 2  # 수정: 출력 크기에 맞게 조정\n# nhead = 8  # 수정: Self-Attention 헤드 수 조정 가능\n","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:10:36.695950Z","iopub.status.idle":"2023-11-24T14:10:36.696289Z","shell.execute_reply.started":"2023-11-24T14:10:36.696132Z","shell.execute_reply":"2023-11-24T14:10:36.696147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 데이터셋을 훈련 및 검증셋으로 분할\n# train_size = int(TRAIN_PROP * len(train_ds))\n# valid_size = len(train_ds) - train_size\n# indices = torch.randperm(len(train_ds))\n# train_sampler = SubsetRandomSampler(indices[:train_size])\n# valid_sampler = SubsetRandomSampler(indices[train_size:])\n\n# # DataLoader 생성\n# train_loader = DataLoader(train_ds, batch_size=BS, sampler=train_sampler, pin_memory=True, num_workers=WORKERS)\n# valid_loader = DataLoader(train_ds, batch_size=1, sampler=valid_sampler, pin_memory=True, num_workers=WORKERS)\n\n# # 모델 초기화\n# model = TransformerEncoder(num_layers, input_size, hidden_size, out_size, nhead).to(device)\n\n# # 옵티마이저, 스케줄러 설정\n# optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=0).to(device)\n# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS, eta_min=0).to(device)\n\n# # 손실 함수\n# criterion = nn.MSELoss()\n# best_loss = 10000000\n# best_model_path = 'best_model.pth'  \n# # 모델 학습 루프\n# for epoch in range(EPOCHS):\n#     model.train()\n#     total_loss = 0\n#     pbar = tqdm(train_loader, desc=f\"Epoch {epoch + 1}/{EPOCHS}\")\n#     for batch_idx, (data, target) in enumerate(pbar):\n#         data = data.float()\n#         target = target.float()\n#         data, target = data.to(device), target.to(device)\n#         optimizer.zero_grad()\n#         output = model(data).to(device)\n#         loss = criterion(output, target)\n#         loss.backward().to(device)\n#         optimizer.step().to(device)\n#         total_loss += loss.item()\n#         pbar.set_postfix({'loss': total_loss / (batch_idx + 1)})\n#     pbar.close()\n\n#     # 검증\n#     model.eval()\n#     with torch.no_grad():\n#         valid_loss = 0\n#         for data, target in valid_loader:\n#             data, target = data.to(device), target.to(device)\n#             output = model(data)\n#             valid_loss += criterion(output, target).item()\n#     if valid_loss < best_loss:\n#         best_loss = valid_loss\n#         torch.save(model.state_dict(), best_model_path)\n\n#     print(f\"Epoch {epoch + 1}/{EPOCHS} - Train Loss: {total_loss / len(train_loader):.4f} - Valid Loss: {valid_loss / len(valid_loader):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:10:36.697665Z","iopub.status.idle":"2023-11-24T14:10:36.698001Z","shell.execute_reply.started":"2023-11-24T14:10:36.697823Z","shell.execute_reply":"2023-11-24T14:10:36.697838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# torch.cuda.empty_cache()\n# torch.cuda.set_per_process_memory_fraction(0.9)","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:10:36.699931Z","iopub.status.idle":"2023-11-24T14:10:36.700302Z","shell.execute_reply.started":"2023-11-24T14:10:36.700140Z","shell.execute_reply":"2023-11-24T14:10:36.700157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n\n# print(pd.__version__)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-24T14:34:18.241102Z","iopub.execute_input":"2023-11-24T14:34:18.241857Z","iopub.status.idle":"2023-11-24T14:34:18.247051Z","shell.execute_reply.started":"2023-11-24T14:34:18.241821Z","shell.execute_reply":"2023-11-24T14:34:18.246045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" ","metadata":{},"execution_count":null,"outputs":[]}]}