{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Standard Imports\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport polars as pl\nimport os\nimport math\nimport random # Use to generate random subjects\nimport time \nimport gc\nfrom tqdm import tqdm\n# Import libraries for working with Parquet files 📄\nfrom pyarrow.parquet import ParquetFile  # PyArrow Parquet file handling 📄\nimport pyarrow as pa          # PyArrow for data serialization 📄\n\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-27T04:47:53.901989Z","iopub.execute_input":"2023-10-27T04:47:53.902333Z","iopub.status.idle":"2023-10-27T04:47:54.595506Z","shell.execute_reply.started":"2023-10-27T04:47:53.902303Z","shell.execute_reply":"2023-10-27T04:47:54.594508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the Configuration and the Filepaths\nclass Paths:\n    # Define data paths here\n    base_path = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    train_events_csv = str(base_path + \"train_events.csv\")\n    train_series_parquet = str(base_path + \"train_series.parquet\")\n    test_series_parquet = str(base_path + \"test_series.parquet\")\n    sample_submission_csv = str(base_path + \"sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:47:54.597265Z","iopub.execute_input":"2023-10-27T04:47:54.597720Z","iopub.status.idle":"2023-10-27T04:47:54.603160Z","shell.execute_reply.started":"2023-10-27T04:47:54.597691Z","shell.execute_reply":"2023-10-27T04:47:54.602197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load the Data \n* Load it efficiently by sampling every nth row","metadata":{}},{"cell_type":"code","source":"class DataLoader():\n    def __init__(self):\n        self.file_props = {\n            \"train_series\": { \"path\": Paths.train_series_parquet, \"is_parquet\": True, \"condense\": True },\n            \"test_series\": { \"path\": Paths.test_series_parquet, \"is_parquet\": True, \"condense\": False },\n            \"train_events\": { \"path\": Paths.train_events_csv, \"is_parquet\": False, \"condense\": False}\n        }\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed rows : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n        # print(data.isna().any())\n        # data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:    \n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    \n    def load_data(self, filepath):\n        \"\"\"\n        Load the data given a filepath\n        \"\"\"\n        file_props = self.file_props[filepath]\n        data = pd.DataFrame()\n        if file_props[\"is_parquet\"]:\n            if file_props[\"condense\"]:\n                pf = ParquetFile(file_props[\"path\"])\n                # Iterate through the parquet file, sampling the data as you go\n                # Read the dataframe in installments of 100000 of the total 127946340\n                # Iterations = ceil(size of total dataset / size of batches)\n                # 127946340 / 100000 = 1280\n                for i in tqdm(range(1_280)):\n                    rows = next(pf.iter_batches(batch_size=100_000))\n                    cur = pa.Table.from_batches([rows]).to_pandas()\n                    # Take a row every 30 seconds. Time step = 5 seconds. Take every 6 row * 5 seconds = 30 seconds\n                    cur = cur[::12*6]\n                    # cur['timestamp'] = pd.to_datetime(cur['timestamp'])\n                    # cur['hour'] = cur['timestamp'].dt.hour\n                    # cur['minute'] = cur['timestamp'].dt.minute\n                    data = pd.concat([data, cur], axis=0).reset_index(drop=True)\n        else:\n            data = pd.read_csv(file_props[\"path\"])\n            \n        gc.collect()\n        if file_props[\"is_parquet\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n            data = self.reduce_memory_usage(data)\n        return data\n# loader = DataLoader()\n# series = loader.load_data(\"train_series\")\n# events = loader.load_data(\"train_events\")","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:47:54.604664Z","iopub.execute_input":"2023-10-27T04:47:54.605183Z","iopub.status.idle":"2023-10-27T04:47:54.625891Z","shell.execute_reply.started":"2023-10-27T04:47:54.605156Z","shell.execute_reply":"2023-10-27T04:47:54.624981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---Data Transforms---\n\n# Transform the datetime column to a year, month, day, hour column \ndatetime_transforms = [\n    pl.col('timestamp').str.to_datetime(),\n    (pl.col('timestamp').str.to_datetime().dt.year()-2000).cast(pl.UInt8).alias('year'),\n    pl.col('timestamp').str.to_datetime().dt.month().cast(pl.UInt8).alias('month'),\n    pl.col('timestamp').str.to_datetime().dt.day().cast(pl.UInt8).alias('day'),\n    pl.col('timestamp').str.to_datetime().dt.hour().cast(pl.UInt8).alias('hour'),\n]\n\n# Transform the enmo, and z-angle data in the train_series dataframe \nseries_data_transforms = [\n    pl.col('anglez').cast(pl.Int16),\n    abs(pl.col('enmo')*1000).cast(pl.UInt16)\n]\n\n# ---Load Series and Event Data---\n# Load series data for train and test, cast the appropriate transforms onto the dataframes\ntrain_series = pl.scan_parquet(Paths.train_series_parquet).with_columns(datetime_transforms + series_data_transforms)\ntest_series = pl.scan_parquet(Paths.test_series_parquet).with_columns(datetime_transforms + series_data_transforms)\n\n# Load event data, cast appropriate transforms\ntrain_events = pl.read_csv(Paths.train_events_csv).with_columns(datetime_transforms)\n\n# Removing series with mismatched event counts (onset vs. wakeup)\nonset_counts = train_events.filter(pl.col('event') == 'onset').group_by('series_id').count().sort('series_id')['count']\nwakeup_counts = train_events.filter(pl.col('event') == 'wakeup').group_by('series_id').count().sort('series_id')['count']\n\n# Getting unique series IDs for convenience\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()\n\ncounts = pl.DataFrame({'series_id': sorted(series_ids), 'onset_counts': onset_counts, 'wakeup_counts': wakeup_counts})\ncount_mismatches = counts.filter(counts['onset_counts'] != counts['wakeup_counts'])\n\n# Filtering out series with count mismatches\ntrain_series = train_series.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))\ntrain_events = train_events.filter(~pl.col('series_id').is_in(count_mismatches['series_id']))\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()\n\n# Get the series_ids from the events dataframe, dropping null values\nseries_ids = train_events['series_id'].unique(maintain_order=True).to_list()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:47:54.628826Z","iopub.execute_input":"2023-10-27T04:47:54.629243Z","iopub.status.idle":"2023-10-27T04:47:54.930134Z","shell.execute_reply.started":"2023-10-27T04:47:54.629199Z","shell.execute_reply":"2023-10-27T04:47:54.929348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ---Feature Engineering---\n\n# Feature data will be aggregated by the hour \nfeatures, feature_cols = [pl.col('enmo'), pl.col('anglez'), pl.col('hour')], ['anglez', 'enmo', 'hour']\n\n# Aggregate the feature data by 30 mins and 120 mins\n# Use mean and max as aggregate functions\n# Explore different aggregation methods in the future\nfor mins in [5, 30, 60*2, 60*4]:\n    features += [\n        pl.col('enmo').rolling_mean(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_mean'),\n        pl.col('enmo').rolling_max(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_max'),\n        pl.col('enmo').rolling_std(12 * mins, center=True, min_periods=1).abs().cast(pl.UInt16).alias(f'enmo_{mins}m_std'),\n    ]\n    \n    # Add new cols to feature columns\n    feature_cols += [\n        f'enmo_{mins}m_mean', f'enmo_{mins}m_max', f'enmo_{mins}m_std'\n    ]\n    \n    for var in ['anglez', 'enmo']:\n        # Aggregate the difference by rolling mean and max\n        features += [\n            (pl.col(var).diff().abs().rolling_mean(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_mean'),\n            (pl.col(var).diff().abs().rolling_max(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_max'),\n            (pl.col(var).diff().abs().rolling_std(12 * mins, center=True, min_periods=1) * 10).abs().cast(pl.UInt32).alias(f'{var}_1v_{mins}m_std')\n        ]\n        \n        feature_cols += [\n            f'{var}_1v_{mins}m_mean', f'{var}_1v_{mins}m_max', f'{var}_1v_{mins}m_std'\n        ]\n        \n# Define the columns used to identify the data to do matching on, included\n# in the series data\nid_cols = ['series_id', 'step', 'timestamp']\n\n# Cast the feature engineering steps onto the train_series and test_series, \n# select all of the columns needed\ntrain_series = train_series.with_columns(\n    features\n).select(id_cols + feature_cols)\n\ntest_series = test_series.with_columns(\n    features\n).select(id_cols + feature_cols + [\"enmo\"])\n\n# ---Takes a sample of the train_series, define x, y values for train set---\n# Collecting datapoints at every 5 minutes\n# data = train_series.filter(pl.col('series_id').is_in(series_ids)).take_every(12 * 5).collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:47:54.932438Z","iopub.execute_input":"2023-10-27T04:47:54.932828Z","iopub.status.idle":"2023-10-27T04:47:54.949258Z","shell.execute_reply.started":"2023-10-27T04:47:54.932790Z","shell.execute_reply":"2023-10-27T04:47:54.948221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/dss-aggregated-data-with-labels-norm/dss_aggregated_data_with_labels_norm.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:47:54.950540Z","iopub.execute_input":"2023-10-27T04:47:54.951729Z","iopub.status.idle":"2023-10-27T04:47:55.969947Z","shell.execute_reply.started":"2023-10-27T04:47:54.951691Z","shell.execute_reply":"2023-10-27T04:47:55.969007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def label_data(series, events):\n    labeled_series = pd.DataFrame()\n    series_ids = series.series_id.unique()\n    \n    for series_id in tqdm(series_ids):\n        asleep = []\n        cur = series[series['series_id'] == series_id].copy()\n        cur_events = events[events['series_id'] == series_id]\n        \n        # Filter \"onset\" and \"wakeup\" events and extract steps\n        onsets = cur_events[(cur_events['event'] == 'onset') & (cur_events['event'] != None)]['step'].to_list()\n        wakeups = cur_events[(cur_events['event'] == 'wakeup') & (cur_events['event'] != None)]['step'].to_list()\n        \n        asleep = sum([(onset <= cur['step']) & (cur['step'] <= wakeup) for onset, wakeup in zip(onsets, wakeups)]).astype(bool)\n        \n        cur['asleep'] = asleep\n        labeled_series = pd.concat([labeled_series, cur])\n    return labeled_series\ntrain_events = train_events.to_pandas()\n# data = label_data(data, train_events)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:49:23.068101Z","iopub.execute_input":"2023-10-27T04:49:23.068588Z","iopub.status.idle":"2023-10-27T04:49:23.085146Z","shell.execute_reply.started":"2023-10-27T04:49:23.068547Z","shell.execute_reply":"2023-10-27T04:49:23.083746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate a random subject from the dataset\ndef random_subject(series):\n    series_ids = series.series_id.unique()\n    \n    ri = random.randrange(0, len(series_ids)-1)\n    \n    subject = series[series['series_id'] == series_ids[ri]]\n    \n    return subject.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:49:25.901455Z","iopub.execute_input":"2023-10-27T04:49:25.901800Z","iopub.status.idle":"2023-10-27T04:49:25.907103Z","shell.execute_reply.started":"2023-10-27T04:49:25.901773Z","shell.execute_reply":"2023-10-27T04:49:25.906048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_daily_graph(series):\n    dates = series['timestamp'].dt.date.unique()\n    events = None\n    \n    def loop():\n        ri = random.randrange(0, len(dates)-1)\n    \n        subject = series[series['timestamp'].dt.date == dates[ri]].reset_index(drop=True)\n    \n        events = train_events[(train_events['series_id'] == subject['series_id'][0]) & (train_events['timestamp'].dt.date == subject['timestamp'].dt.date[0])]\n        return subject, events\n    \n    subject, events = loop()\n    if events.empty:\n        while events.empty:\n            subject, events = loop()\n\n    return subject.reset_index(drop=True), events\n\ntrain_events['timestamp'] = pd.to_datetime(train_events['timestamp'])","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:49:26.147918Z","iopub.execute_input":"2023-10-27T04:49:26.148288Z","iopub.status.idle":"2023-10-27T04:49:26.169067Z","shell.execute_reply.started":"2023-10-27T04:49:26.148258Z","shell.execute_reply":"2023-10-27T04:49:26.168224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subject = random_subject(data)\nday, events = generate_daily_graph(subject)\n\n# Get the events for the day\n# events = train_events[(train_events['series_id'] == day['series_id'][0]) & (train_events['timestamp'].dt.date == day['timestamp'].dt.date[0])]\nonset = events[events['event'] == \"onset\"]['step'].iloc[0]\nwakeup = events[events['event'] == \"wakeup\"]['step'].iloc[0]\n\n# Create a figure and an axis\nfig, ax = plt.subplots(1, 1, figsize=(12, 4))  # 1 row, 3 columns\ncat = 'anglez_1v_5m_mean'\nX, y = day['step'], day[cat]\n\nplt.plot(X, y)\n# Define the label coordinates for onset and wakeup\nlabel_onset_x, label_onset_y = onset, day[(day['step'] > onset - 60) & (day['step'] < onset + 60)][cat].mean() \nlabel_wakeup_x, label_wakeup_y = wakeup, day[(day['step'] > wakeup - 60) & (day['step'] < wakeup + 60)][cat].mean()\n\n# Add dots for onset and wakeup and annotate\nax.scatter(label_onset_x, label_onset_y, color='red', marker='o', s=100, label='Onset')\nax.scatter(label_wakeup_x, label_wakeup_y, color='red', marker='o', s=100, label='Wakeup')\nplt.title(f'Subject: {day[\"series_id\"][0]}, date: {day[\"timestamp\"].dt.date[0]}, category: {cat}')\nplt.xlabel(f'Step')\nplt.ylabel(f'{cat}')","metadata":{"execution":{"iopub.status.busy":"2023-10-27T04:50:25.690888Z","iopub.execute_input":"2023-10-27T04:50:25.691282Z","iopub.status.idle":"2023-10-27T04:50:26.193001Z","shell.execute_reply.started":"2023-10-27T04:50:25.691251Z","shell.execute_reply":"2023-10-27T04:50:26.191871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}