{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-26T04:47:09.788724Z","iopub.execute_input":"2023-10-26T04:47:09.789122Z","iopub.status.idle":"2023-10-26T04:47:10.244202Z","shell.execute_reply.started":"2023-10-26T04:47:09.789087Z","shell.execute_reply":"2023-10-26T04:47:10.242881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\nimport time\nimport json\nimport math\n\nfrom math import pi\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\n\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes\ntorch.set_num_interop_threads(4)\ntorch.set_num_threads(4)\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:10.246295Z","iopub.execute_input":"2023-10-26T04:47:10.246749Z","iopub.status.idle":"2023-10-26T04:47:14.477321Z","shell.execute_reply.started":"2023-10-26T04:47:10.246719Z","shell.execute_reply":"2023-10-26T04:47:14.475661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\nclass CFG:\n    DEMO_MODE = True\nclass data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed rows : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_rows = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_rows]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nrows=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:14.479115Z","iopub.execute_input":"2023-10-26T04:47:14.480680Z","iopub.status.idle":"2023-10-26T04:47:14.536016Z","shell.execute_reply.started":"2023-10-26T04:47:14.480643Z","shell.execute_reply":"2023-10-26T04:47:14.534943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nreader = data_reader(demo_mode=False)\ntest_data = reader.load_data(data_name=\"test_series\")\ntest_data = test_data.drop('timestamp',axis=1)\ntest_data = test_data.drop('anglez',axis=1)\n\nids = test_data.series_id.unique()\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:14.539705Z","iopub.execute_input":"2023-10-26T04:47:14.540955Z","iopub.status.idle":"2023-10-26T04:47:15.077685Z","shell.execute_reply.started":"2023-10-26T04:47:14.540909Z","shell.execute_reply":"2023-10-26T04:47:15.076455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_900_forward_moving_averages(array):\n    fma900 = np.convolve(array, np.ones(900) /900, mode='full')\n    return fma900[899:len(array)+900]\n\ndef calculate_120_forward_moving_averages(array):\n    fma120 = np.convolve(array, np.ones(120) /120, mode='full')\n    return fma120[119:len(array)+120]\n# Function to calculate moving averages\ndef calculate_12_moving_averages(array):\n    ma12 = np.convolve(array, np.ones(12)/12, mode='same')\n    return ma12","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:15.079648Z","iopub.execute_input":"2023-10-26T04:47:15.080483Z","iopub.status.idle":"2023-10-26T04:47:15.089216Z","shell.execute_reply.started":"2023-10-26T04:47:15.080441Z","shell.execute_reply":"2023-10-26T04:47:15.088385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to set the new column's value\n#Calculated mean when onset\n# mean 12ma :  0.022531045538178317\n# mean 120ma:  0.009181115043061284\n# mean 900ma:  0.007875275758680782\n#Calculated mean when wakeup\n# mean 12ma:  0.006950316923481253\n# mean 120ma:  0.03177788946894143\n# mean 900ma:  0.048272512918419774\ndef set_condition(ma12,fma120,fma900):\n\n    start = 0\n    end = 0\n    if (ma12 < 0.0225):\n        if(fma120 < 0.00918):\n            if(fma900 < 0.00787):\n                if(fma900 > 0.0004):\n                    if(fma120 > 0.002):\n                        if (ma12 > 0.002):\n                            start = 1\n    if(ma12 > 0.045):\n        if(fma120 > 0.032):\n            if(fma900 > 0.048):\n                end = 1\n    return start, end","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:15.091000Z","iopub.execute_input":"2023-10-26T04:47:15.091817Z","iopub.status.idle":"2023-10-26T04:47:15.111801Z","shell.execute_reply.started":"2023-10-26T04:47:15.091776Z","shell.execute_reply":"2023-10-26T04:47:15.110954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Initialize state and new column\nsubmission = []\ni=0\nfor idx, group in test_data.groupby('series_id'):\n    array = group['enmo'].to_numpy()\n    # Calculate moving averages\n    ma12 = calculate_12_moving_averages(array)\n    fma120 = calculate_120_forward_moving_averages(array)\n    fma900 = calculate_900_forward_moving_averages(array)\n    group_series = group.reset_index(drop=True)\n    seriesName = group_series.iloc[0]['series_id']\n    state = 0\n    for idx in range (len(group_series)):\n        start_condition, end_condition = set_condition(ma12[idx],fma120[idx],fma900[idx])\n        if start_condition and not state:\n            OnsetStepValue = group_series.iloc[idx]['step']\n            state = 1\n        if end_condition and state:\n            WakeupStepValue = group_series.iloc[idx]['step']\n            state = 0\n            if(2880 < WakeupStepValue-OnsetStepValue < 8640): #More than 4 hours and less than 12 hours\n                submission.append({'row_id': i, 'series_id': seriesName, 'step': OnsetStepValue, 'event': 'onset', 'score': 0})\n                submission.append({'row_id': i+1, 'series_id': seriesName, 'step': WakeupStepValue, 'event': 'wakeup', 'score': 0})\n                i += 2\n# Convert the list of dictionaries to a DataFrame\nsubmission_pd = pd.DataFrame(submission)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:15.113696Z","iopub.execute_input":"2023-10-26T04:47:15.114550Z","iopub.status.idle":"2023-10-26T04:47:15.140336Z","shell.execute_reply.started":"2023-10-26T04:47:15.114510Z","shell.execute_reply":"2023-10-26T04:47:15.139196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_pd.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-26T04:47:15.142257Z","iopub.execute_input":"2023-10-26T04:47:15.143400Z","iopub.status.idle":"2023-10-26T04:47:15.153321Z","shell.execute_reply.started":"2023-10-26T04:47:15.143360Z","shell.execute_reply":"2023-10-26T04:47:15.152142Z"},"trusted":true},"execution_count":null,"outputs":[]}]}