{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Detect Sleep States DataPrepare\n\n### his notebook is inspired by the following [werus23's](https://www.kaggle.com/werus23) amazing notebooks.\n### see also them.\n\nFinding Critical Points using RNNs\n\n - [Data Prep notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-prepare-data)\n - [Training notebook](https://www.kaggle.com/werus23/sleep-critical-point-train)\n - [Inference Notebook](https://www.kaggle.com/code/werus23/sleep-critical-point-infer)\n\nCredits:\n - dataloader: https://www.kaggle.com/code/henriupton/efficient-loading-memory-usage-visualizations-cmi","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm \n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:09:26.950407Z","iopub.execute_input":"2023-10-22T05:09:26.950922Z","iopub.status.idle":"2023-10-22T05:09:34.740822Z","shell.execute_reply.started":"2023-10-22T05:09:26.950879Z","shell.execute_reply":"2023-10-22T05:09:34.739869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:09:34.742537Z","iopub.execute_input":"2023-10-22T05:09:34.742871Z","iopub.status.idle":"2023-10-22T05:09:34.748492Z","shell.execute_reply.started":"2023-10-22T05:09:34.742843Z","shell.execute_reply":"2023-10-22T05:09:34.746856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_dir = 'train_csvs'\nos.makedirs(out_dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:09:34.749585Z","iopub.execute_input":"2023-10-22T05:09:34.750185Z","iopub.status.idle":"2023-10-22T05:09:34.763645Z","shell.execute_reply.started":"2023-10-22T05:09:34.750157Z","shell.execute_reply":"2023-10-22T05:09:34.762578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed steps : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_steps = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_steps]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nsteps=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        #data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:09:34.766197Z","iopub.execute_input":"2023-10-22T05:09:34.766777Z","iopub.status.idle":"2023-10-22T05:09:34.792679Z","shell.execute_reply.started":"2023-10-22T05:09:34.766745Z","shell.execute_reply":"2023-10-22T05:09:34.791336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"train_series\")\nevents = reader.load_data(data_name=\"train_events\")","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:09:34.794181Z","iopub.execute_input":"2023-10-22T05:09:34.794537Z","iopub.status.idle":"2023-10-22T05:11:03.558786Z","shell.execute_reply.started":"2023-10-22T05:09:34.794505Z","shell.execute_reply":"2023-10-22T05:11:03.557701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIGMA = 720 # 12 * 60\ndef gauss(n=SIGMA,sigma=SIGMA*0.15):\n    # guassian distribution function\n    r = range(-int(n/2),int(n/2)+1)\n    return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:11:03.560030Z","iopub.execute_input":"2023-10-22T05:11:03.560349Z","iopub.status.idle":"2023-10-22T05:11:03.567035Z","shell.execute_reply.started":"2023-10-22T05:11:03.560322Z","shell.execute_reply":"2023-10-22T05:11:03.565859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\nids = series.series_id.unique()\n\nenmo_dfs = pd.DataFrame()\n\nfor j, viz_id in tqdm(enumerate(ids), total=len(ids)):\n    viz_targets = []\n    viz_events = events[events.series_id == viz_id]\n    viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n    viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n    viz_series['hour'] = viz_series['dt'].dt.hour\n\n    check = 0\n    for i in range(len(viz_events)-1):\n        if viz_events.iloc[i].event =='onset' and viz_events.iloc[i+1].event =='wakeup' and viz_events.iloc[i].night==viz_events.iloc[i+1].night:\n            start,end = viz_events.timestamp.iloc[i],viz_events.timestamp.iloc[i+1]\n\n            start_id = viz_series.loc[viz_series.timestamp ==start].index.values[0]\n            end_id = viz_series.loc[viz_series.timestamp ==end].index.values[0]\n            viz_targets.append((start_id,end_id))\n            check+=1\n\n    target_guassian = np.zeros((len(viz_series),2))\n\n    for s,e in viz_targets:\n        st1,st2 = max(0,s-SIGMA//2),s+SIGMA//2+1\n        ed1,ed2 = e-SIGMA//2,min(len(viz_series),e+SIGMA//2+1)\n        target_guassian[st1:st2,0] = gauss()[st1-(s-SIGMA//2):]\n        target_guassian[ed1:ed2,1] = gauss()[:SIGMA+1-((e+SIGMA//2+1)-ed2)]\n\n    target_guassian /= np.max(target_guassian + 1e-12)\n\n    viz_series['onset'] = target_guassian[:,0]\n    viz_series['wakeup'] = target_guassian[:,1]\n\n    new_df = viz_series[['step', 'anglez', 'enmo', 'hour', 'onset', 'wakeup']]\n    new_df.to_csv(f'{out_dir}/{viz_id}.csv', index=False)\n\n    enmo_dfs = pd.concat([enmo_dfs, new_df] ,axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:11:03.568424Z","iopub.execute_input":"2023-10-22T05:11:03.568818Z","iopub.status.idle":"2023-10-22T05:13:44.269989Z","shell.execute_reply.started":"2023-10-22T05:11:03.568789Z","shell.execute_reply":"2023-10-22T05:13:44.268182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_dfs = enmo_dfs.reset_index()\nenmo_dfs.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:13:46.684150Z","iopub.execute_input":"2023-10-22T05:13:46.684560Z","iopub.status.idle":"2023-10-22T05:13:46.747233Z","shell.execute_reply.started":"2023-10-22T05:13:46.684527Z","shell.execute_reply":"2023-10-22T05:13:46.746222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enmo_mean = enmo_dfs['enmo'].mean()\nnp.save('enmo_mean.npy', enmo_mean)\nenmo_std = enmo_dfs['enmo'].std()\nnp.save('enmo_std.npy', enmo_std)\nprint(f'mean:{enmo_mean:.6f}, std:{enmo_std:.6f}')","metadata":{"execution":{"iopub.status.busy":"2023-10-22T05:13:46.851794Z","iopub.execute_input":"2023-10-22T05:13:46.852177Z","iopub.status.idle":"2023-10-22T05:13:46.877577Z","shell.execute_reply.started":"2023-10-22T05:13:46.852148Z","shell.execute_reply":"2023-10-22T05:13:46.876794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}