{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Efficient Loading+Memory Usage+Visualizations⚡CMI Competition","metadata":{}},{"cell_type":"markdown","source":"This notebook is dedicated to data loading for CMI (Child Mind Institute) for the competition \"Detect Sleep States\".\nThis is actually version 1 of the notebook. \n\n*Important informations about the notebook* : \n- it allows to load the data thanks to a class named **data_reader** that handles memory usage issues for large files such as **train_series**.\n- a **demo mode** is available, it allows to load only a mini match of the full data for fast loading/training. Its only for temporarly work, please disable this demo mode if you want to load the full data (by setting DEMO_MODE = False inside of the CONFIG class)\n- first **visualization tools** are included for data, such as time series, markups of sleeping intervals (onset & wakeup)\n\nWill be included in the future :\n- **Visualizer class** instance in order obtain visualization for an input night id and series id\n- **PyTorch module dataset** to load data for training purposes\n\nDon't hesitate to leave feedback and upvote for support 👌👌","metadata":{}},{"cell_type":"markdown","source":"# 1. Configuration","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport matplotlib.pyplot as plt\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-08T16:27:24.404151Z","iopub.execute_input":"2023-09-08T16:27:24.404828Z","iopub.status.idle":"2023-09-08T16:27:29.069420Z","shell.execute_reply.started":"2023-09-08T16:27:24.404781Z","shell.execute_reply":"2023-09-08T16:27:29.068373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:31.645833Z","iopub.execute_input":"2023-09-08T16:27:31.646369Z","iopub.status.idle":"2023-09-08T16:27:31.653288Z","shell.execute_reply.started":"2023-09-08T16:27:31.646337Z","shell.execute_reply":"2023-09-08T16:27:31.652057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEMO_MODE = True\n    VERBOSE = True\n    \n    BATCH_SIZE = 32\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:32.257809Z","iopub.execute_input":"2023-09-08T16:27:32.258222Z","iopub.status.idle":"2023-09-08T16:27:32.264345Z","shell.execute_reply.started":"2023-09-08T16:27:32.258189Z","shell.execute_reply":"2023-09-08T16:27:32.262826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. *Global Data Loader* instance","metadata":{}},{"cell_type":"code","source":"class CMIDataReader:\n    def __init__(self, demo_mode, verbose):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n        self.verbose = verbose\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        if self.verbose:\n            print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        if self.verbose:\n            print(\"Percentage of removed rows : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n        data = data.bfill()\n        return data\n    \n    def reduce_memory_usage(self, data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        if self.verbose:\n            print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        if self.verbose:\n            print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n            print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_rows = next(pf.iter_batches(batch_size=100_000)) \n                data = pa.Table.from_batches([demo_rows]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nrows=100_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n        \n        if data_props[\"has_timestamp\"]:\n            data = self.cleaning(data)\n            data[\"timestamp\"] = pd.to_datetime(data[\"timestamp\"])\n            data = data.set_index(\"timestamp\")\n        \n        data = self.reduce_memory_usage(data)\n        gc.collect()\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:33.778483Z","iopub.execute_input":"2023-09-08T16:27:33.779569Z","iopub.status.idle":"2023-09-08T16:27:33.816162Z","shell.execute_reply.started":"2023-09-08T16:27:33.779520Z","shell.execute_reply":"2023-09-08T16:27:33.815214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = CMIDataReader(demo_mode=CFG.DEMO_MODE, verbose=CFG.VERBOSE)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:34.528140Z","iopub.execute_input":"2023-09-08T16:27:34.528974Z","iopub.status.idle":"2023-09-08T16:27:34.535274Z","shell.execute_reply.started":"2023-09-08T16:27:34.528935Z","shell.execute_reply":"2023-09-08T16:27:34.533668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"=\"*25)\nprint(\"LOAD SUBMISSION\")\nsubmission = reader.load_data(\"submission\")\nprint(submission.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"LOAD TEST SERIES\")\ntest_series = reader.load_data(\"test_series\")\nprint(test_series.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"=\"*25)\nprint(\"LOAD TRAIN SERIES\")\ntrain_series = reader.load_data(\"train_series\")\nprint(train_series.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)\n\nprint(\"=\"*25)\nprint(\"LOAD TRAIN EVENTS\")\ntrain_events = reader.load_data(\"train_events\")\nprint(train_events.head(10))\nprint(\"LOADING FINISHED ! \")\nprint(\"=\"*25)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:52:34.568685Z","iopub.execute_input":"2023-09-08T16:52:34.569162Z","iopub.status.idle":"2023-09-08T16:52:38.050983Z","shell.execute_reply.started":"2023-09-08T16:52:34.569126Z","shell.execute_reply":"2023-09-08T16:52:38.049061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. *Visualizer* instance","metadata":{}},{"cell_type":"code","source":"class CMIDataVisualizer:\n    \n    def __init__(self, series_data, events_data):\n        self.series_data = series_data\n        self.events_data = events_data\n    \n    def plot_events(self, series_id, night_id):\n        query_series = self.series_data[self.series_data.series_id == series_id]\n        query_events = self.events_data[(self.events_data.series_id == series_id) \\\n                                       & (self.events_data.night == night_id)][\"event\"]\n        \n        _, axes = plt.subplots(2,1, figsize = (20,8))\n        for t in range(len(query_events)):\n            if query_events.values[t] == \"onset\":\n                axes[1].axvline(query_events.index[t], linestyle=\"--\", color=\"green\")\n                axes[1].annotate(\n                    \"Onset: {}\".format(query_events.index[t].strftime(\"%Y-%m-%d %H:%M:%S\")),\n                    xy=(query_events.index[t], axes[1].get_ylim()[1]),\n                    xytext=(10, 10),\n                    textcoords=\"offset points\",\n                    fontsize=10,\n                    color=\"green\",\n                )\n\n            if query_events.values[t] == \"wakeup\":\n                axes[1].axvline(query_events.index[t], linestyle=\"--\", color=\"blue\")\n                axes[1].annotate(\n                    \"Wakeup: {}\".format(query_events.index[t].strftime(\"%Y-%m-%d %H:%M:%S\")),\n                    xy=(query_events.index[t], axes[1].get_ylim()[1]),\n                    xytext=(10, 10),\n                    textcoords=\"offset points\",\n                    fontsize=10,\n                    color=\"blue\",\n                )\n        axes[0].plot(train_series.enmo)\n        axes[0].set_ylabel(\"enmo\")\n        axes[1].plot(train_series.anglez)\n        axes[1].set_ylabel(\"anglez\")\n        axes[1].legend()\n        axes[0].set_title(\"series id : {} / night id : {}\".format(series_id, night_id), size=20)\n        plt.legend()\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:44.166348Z","iopub.execute_input":"2023-09-08T16:27:44.166941Z","iopub.status.idle":"2023-09-08T16:27:44.185333Z","shell.execute_reply.started":"2023-09-08T16:27:44.166898Z","shell.execute_reply":"2023-09-08T16:27:44.183917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualizer = CMIDataVisualizer(\n    series_data=train_series,\n    events_data=train_events\n)\n\nseries_id = train_series.series_id.unique()[0]\nfor night_id in range(1,6):\n    visualizer.plot_events(\n        series_id=series_id,\n        night_id=night_id\n    )","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:27:44.722354Z","iopub.execute_input":"2023-09-08T16:27:44.723621Z","iopub.status.idle":"2023-09-08T16:28:09.309721Z","shell.execute_reply.started":"2023-09-08T16:27:44.723568Z","shell.execute_reply":"2023-09-08T16:28:09.308559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. *Dataset* + *Dataloader* Pytorch instances","metadata":{}},{"cell_type":"markdown","source":"Important remarks about training features + targets : \n- all records of events and series has a common **step** column as time measures : the first timestamp of the dataset is the starting point of the \"step\" measure. Then every 5 seconds in the timestamps the \"step\" get incremented by one. For example step=0 for timestamp=2018-08-14 15:30:00-04:00. Then step=1 for timestamp=2018-08-14 15:30:05-04:00, and so on.\n- as we saw from data loading + visualization procedures, we originally dispose of \"train events\" that indicates for each single **night** when the patient starts the sleep session (**onset**) and ends the sleep session (**wakeup**). We dispose of the corresponding **step** value for each **onset** and **wakeup** events. For example, the first **onset** event is observed at timstamp=2018-08-14 22:26:00-04:00 for a value of step=4992. Indeed, 22:26:00 = 15:30:00 + 5 seconds x 4992\n- in reality for training, we need to associate an event between **onset** or **wakeup** for each observed step. We are able to do this by using the events data : for each step before the first observed **onset** event, the patient is **wakeup** (for our case for each event before step=4992). Then for all steps between the first event **onset** and the first event **wakeup**, the patient is **onset**. Then between the first event **wakeup** and the second event **onset** (second night), the patient is **wakeup**. And so on for all the nights. Technically, we can do a left merge between train_series and train_events in order to keep all steps and their associated features, and to get the labels for the particular events steps. Then we will be able to fill NaN values with forward fill, that basically fill NaN values by the last observed value of the column (which is exactly the principle that we explained before). The only particular case is for the steps before the first event (onset) that we need to fill with the \"wakeup\" value.  ","metadata":{}},{"cell_type":"code","source":"print(train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T16:52:58.217389Z","iopub.execute_input":"2023-09-08T16:52:58.217824Z","iopub.status.idle":"2023-09-08T16:52:58.231832Z","shell.execute_reply.started":"2023-09-08T16:52:58.217791Z","shell.execute_reply":"2023-09-08T16:52:58.230442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"full_data = train_series.merge(train_events[[\"event\",\"step\"]],on=\"step\",how=\"left\")\nprint(full_data)\nfull_data[\"event\"] = full_data[\"event\"].ffill()\nprint(full_data)\nfull_data[\"event\"] = full_data[\"event\"].fillna(\"wakeup\")\nprint(full_data)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T17:01:06.628067Z","iopub.execute_input":"2023-09-08T17:01:06.628474Z","iopub.status.idle":"2023-09-08T17:01:06.686113Z","shell.execute_reply.started":"2023-09-08T17:01:06.628428Z","shell.execute_reply":"2023-09-08T17:01:06.684913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CMITrainDataset(Dataset):\n    \n    def __init__(self, series_data, events_data):\n        full_data = series_data.merge(events_data[[\"event\",\"step\"]],on=\"step\",how=\"left\")\n        full_data[\"event\"] = full_data[\"event\"].ffill().fillna(\"wakeup\")\n        full_data[\"event\"] = full_data[\"event\"].map({\"wakeup\":0, \"onset\":1})\n        \n        self.features = full_data[[\"anglez\",\"enmo\"]]\n        self.targets = full_data[\"event\"]\n    \n    def __len__(self):\n        return len(self.targets)\n    \n    def __getitem__(self, index):\n        features = torch.tensor(self.features.iloc[index], dtype = torch.float32)\n        target = torch.tensor(self.targets.iloc[index], dtype = torch.int64)\n        return features, target","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CMITrainDataset(\n    series_data = train_series, \n    events_data = train_events\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.__getitem__(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=CFG.BATCH_SIZE, shuffle=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"next(iter(train_dataloader))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"https://www.icegif.com/wp-content/uploads/2022/06/icegif-1101.gif\">","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-09-07T11:37:22.218736Z","iopub.execute_input":"2023-09-07T11:37:22.219221Z","iopub.status.idle":"2023-09-07T11:37:22.227118Z","shell.execute_reply.started":"2023-09-07T11:37:22.219180Z","shell.execute_reply":"2023-09-07T11:37:22.224963Z"}}}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}