{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Child Mind Institute - Detect Sleep States\n\nCredits:\n\n - https://www.kaggle.com/code/itsuki9180/detect-sleep-states-dataprepare","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm \n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:14.293908Z","iopub.status.busy":"2023-11-21T19:13:14.293512Z","iopub.status.idle":"2023-11-21T19:13:22.241333Z","shell.execute_reply":"2023-11-21T19:13:22.239619Z","shell.execute_reply.started":"2023-11-21T19:13:14.293877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:22.244612Z","iopub.status.busy":"2023-11-21T19:13:22.243858Z","iopub.status.idle":"2023-11-21T19:13:22.251902Z","shell.execute_reply":"2023-11-21T19:13:22.250105Z","shell.execute_reply.started":"2023-11-21T19:13:22.244567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_dir = 'train_csvs'\nos.makedirs(out_dir, exist_ok=True)","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:22.254203Z","iopub.status.busy":"2023-11-21T19:13:22.253760Z","iopub.status.idle":"2023-11-21T19:13:22.265844Z","shell.execute_reply":"2023-11-21T19:13:22.264952Z","shell.execute_reply.started":"2023-11-21T19:13:22.254161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", data.filter(pl.col(\"timestamp\").is_null()).height)\n        data = data.filter(pl.col(\"timestamp\").is_not_null())\n        after_cleaning = len(data)\n        print(\"Percentage of removed steps : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n        return data\n\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n\n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                # Polars doesn't directly support reading specific batches from Parquet as Pandas does\n                # You need to read the full file and then slice it, which might not be efficient in demo mode\n                data = pl.read_parquet(data_props[\"path\"]).limit(20_000)\n            else:\n                data = pl.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                # Polars also supports reading a specific number of rows from a CSV\n                data = pl.read_csv(data_props[\"path\"]).limit(20_000)\n            else:\n                data = pl.read_csv(data_props[\"path\"])\n\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n        return data\n    ","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:22.270250Z","iopub.status.busy":"2023-11-21T19:13:22.269354Z","iopub.status.idle":"2023-11-21T19:13:22.301853Z","shell.execute_reply":"2023-11-21T19:13:22.300618Z","shell.execute_reply.started":"2023-11-21T19:13:22.270194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"train_series\")\nevents = reader.load_data(data_name=\"train_events\")","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:22.305648Z","iopub.status.busy":"2023-11-21T19:13:22.305210Z","iopub.status.idle":"2023-11-21T19:13:42.563075Z","shell.execute_reply":"2023-11-21T19:13:42.561721Z","shell.execute_reply.started":"2023-11-21T19:13:22.305606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIGMA = 720 # 12 * 60\ndef gauss(n=SIGMA,sigma=SIGMA*0.15):\n    # guassian distribution function\n    r = range(-int(n/2),int(n/2)+1)\n    return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:42.565386Z","iopub.status.busy":"2023-11-21T19:13:42.564940Z","iopub.status.idle":"2023-11-21T19:13:42.572567Z","shell.execute_reply":"2023-11-21T19:13:42.571419Z","shell.execute_reply.started":"2023-11-21T19:13:42.565343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Gaussian 함수 정의\ndef gauss(n=SIGMA, sigma=SIGMA*0.15):\n    r = range(-int(n/2), int(n/2) + 1)\n    return [1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(-float(x)**2 / (2 * sigma**2)) for x in r]\n","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:42.574183Z","iopub.status.busy":"2023-11-21T19:13:42.573772Z","iopub.status.idle":"2023-11-21T19:13:42.586972Z","shell.execute_reply":"2023-11-21T19:13:42.585655Z","shell.execute_reply.started":"2023-11-21T19:13:42.574144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\n# ids = series.series_id.unique()\nids = series.select('series_id').unique()\n\nenmo_dfs = pl.DataFrame()","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:42.588992Z","iopub.status.busy":"2023-11-21T19:13:42.588552Z","iopub.status.idle":"2023-11-21T19:13:46.157279Z","shell.execute_reply":"2023-11-21T19:13:46.156020Z","shell.execute_reply.started":"2023-11-21T19:13:42.588952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for viz_id in tqdm(ids[\"series_id\"]):\n    viz_series = series.filter(pl.col(\"series_id\") == viz_id)\n    viz_events = events.filter(pl.col(\"series_id\") == viz_id)\n\n    # Datetime 처리 및 새로운 컬럼 추가\n    viz_series = viz_series.with_columns(\n        pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\").alias(\"dt\")\n    ).with_columns(\n        pl.col(\"dt\") - pl.duration(hours=4)\n    ).with_columns([\n        pl.col(\"dt\").dt.hour().alias(\"hour\")\n    ])\n    \n    # 'onset' 및 'wakeup' 이벤트 타임스탬프 찾기\n    viz_targets = []\n    target_gaussian = np.zeros((len(viz_series), 2))\n    for i in range(len(viz_events) - 1):\n        if viz_events[i, \"event\"] == \"onset\" and viz_events[i + 1, \"event\"] == \"wakeup\" and viz_events[i, \"night\"] == viz_events[i + 1, \"night\"]:\n            start_time, end_time = viz_events[i, \"timestamp\"], viz_events[i + 1, \"timestamp\"]\n            start_id = viz_series.with_row_count().filter(pl.col(\"timestamp\") == start_time).item(0,0)\n            end_id = viz_series.with_row_count().filter(pl.col(\"timestamp\") == end_time).item(0,0)\n            viz_targets.append((start_id, end_id))\n            \n    # 각 행에 가우시안 분포 적용\n    for s,e in viz_targets:\n        st1,st2 = max(0,s-SIGMA//2),s+SIGMA//2+1\n        ed1,ed2 = e-SIGMA//2,min(len(viz_series),e+SIGMA//2+1)\n        target_gaussian[st1:st2,0] = gauss()[st1-(s-SIGMA//2):]\n        target_gaussian[ed1:ed2,1] = gauss()[:SIGMA+1-((e+SIGMA//2+1)-ed2)]\n\n\n    # 가우시안 분포 정규화 및 데이터 프레임에 추가\n    target_gaussian /= np.max(target_gaussian + 1e-12)\n    viz_series = viz_series.with_columns([\n        pl.lit(target_gaussian[:, 0]).alias(\"onset\"),\n        pl.lit(target_gaussian[:, 1]).alias(\"wakeup\")\n    ])\n\n#     print(viz_id)\n    # 결과 데이터 프레임 저장\n    new_df = viz_series.select(['step', 'anglez', 'enmo', 'hour', 'onset', 'wakeup'])  # onset, wakeup 추가 필요\n    # Polars에서는 to_csv 메서드로 CSV 파일을 저장할 수 있음\n    new_df.write_csv(f'{out_dir}/{viz_id}.csv')\n\n    # 데이터 프레임 병합\n    enmo_dfs = pl.concat([enmo_dfs, new_df])\n","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:13:46.158897Z","iopub.status.busy":"2023-11-21T19:13:46.158557Z","iopub.status.idle":"2023-11-21T19:33:08.259346Z","shell.execute_reply":"2023-11-21T19:33:08.257311Z","shell.execute_reply.started":"2023-11-21T19:13:46.158869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_df = enmo_dfs.to_pandas()\nenmo_df = enmo_df.reset_index()  # 인덱스를 재설정\nenmo_df.head()  # 처음 몇 개의 행을 출력","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:33:08.265749Z","iopub.status.busy":"2023-11-21T19:33:08.265332Z","iopub.status.idle":"2023-11-21T19:33:16.051535Z","shell.execute_reply":"2023-11-21T19:33:16.049168Z","shell.execute_reply.started":"2023-11-21T19:33:08.265713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_mean = enmo_df['enmo'].mean()\nnp.save('enmo_mean.npy', enmo_mean)\nenmo_std = enmo_df['enmo'].std()\nnp.save('enmo_std.npy', enmo_std)\nprint(f'mean:{enmo_mean:.6f}, std:{enmo_std:.6f}')","metadata":{"execution":{"iopub.execute_input":"2023-11-21T19:33:16.056357Z","iopub.status.busy":"2023-11-21T19:33:16.054791Z","iopub.status.idle":"2023-11-21T19:33:17.786901Z","shell.execute_reply":"2023-11-21T19:33:17.785715Z","shell.execute_reply.started":"2023-11-21T19:33:16.056252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}