{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"*# Create a smaller \"starter\" datasets + a target column called `awake`\n\nIn this competition we are given 277 series. However, for 8 of these series we have absolutely no event data (`onset` or `wakeup` timestamps). Indeed, there are only 37 series of events data for which there are no missing events at all, and two of these seem to be truncated. \n\nWithout the corresponding event data it is particularly hard to untangle the sleeping from the awake data contained in the `train_series` data. With this in mind here we create a smaller training dataset consisting of a selection of just 35 of the series. We also add a binary target column called `awake`.\n\n(For plots of all 277 series see the notebook [\"Zzzs: Creating a \"target\" column + viz of all data\"](https://www.kaggle.com/code/carlmcbrideellis/zzzs-creating-a-target-column-viz-of-all-data))\n\n# `Zzzs_train.parquet`\nBinary dataset (`awake = 1, sleeping = 0`,'onset'=6,'wakeup'=7)\n\nthis is a smaller starter processed dataset for experimenting with tensorflow\nTO DO: expand the dataset with awake windows arounds onset wake-up events","metadata":{}},{"cell_type":"markdown","source":"\"sleep\"=0,\"awake\"=1,\"onset\"=2,\"wakeup\"=3","metadata":{}},{"cell_type":"code","source":"import random\nimport string\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:03:53.169236Z","iopub.execute_input":"2023-10-25T19:03:53.170184Z","iopub.status.idle":"2023-10-25T19:03:53.209754Z","shell.execute_reply.started":"2023-10-25T19:03:53.170145Z","shell.execute_reply":"2023-10-25T19:03:53.208632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Numpy Dataset","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nfrom os.path import isfile, join\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import Markdown\nimport gc\nimport numpy as np\nimport pickle\nfrom scipy import signal\nfrom sklearn.preprocessing import MinMaxScaler\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport sys\nfrom IPython.display import Markdown\nsns.set_style(\"whitegrid\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:03:53.211754Z","iopub.execute_input":"2023-10-25T19:03:53.212125Z","iopub.status.idle":"2023-10-25T19:03:55.867018Z","shell.execute_reply.started":"2023-10-25T19:03:53.212093Z","shell.execute_reply":"2023-10-25T19:03:55.865977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = 'dataset/signals120'\n\n!mkdir {HOME_DIR}dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/train\npath_to_train=\"/kaggle/working/dataset/signals120/train/\"\n!mkdir {HOME_DIR}{DATASET_PATH}/val\npath_to_test=\"/kaggle/working/dataset/signals120/val/\"\n!mkdir {HOME_DIR}{DATASET_PATH}/complete\n\n!mkdir {HOME_DIR}{DATASET_PATH}/real\npath_to_real=\"/kaggle/working/dataset/signals120/real/\"\ndatadir=\"/kaggle/working/dataset/signals120/complete/\"\npath_to_train=\"../working/dataset/signals120/train/\"\npath_to_test=\"../working/dataset/signals120/val/\"\ndatadir","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:03:55.868633Z","iopub.execute_input":"2023-10-25T19:03:55.869485Z","iopub.status.idle":"2023-10-25T19:04:02.660121Z","shell.execute_reply.started":"2023-10-25T19:03:55.869446Z","shell.execute_reply":"2023-10-25T19:04:02.658642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import module\n\n# path_to_train=\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train.parquet\"\n#path_to_train=\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\"\n# create data\ndata = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:04:02.664407Z","iopub.execute_input":"2023-10-25T19:04:02.664836Z","iopub.status.idle":"2023-10-25T19:04:58.577603Z","shell.execute_reply.started":"2023-10-25T19:04:02.664801Z","shell.execute_reply":"2023-10-25T19:04:58.576507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.columns","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:04:58.578590Z","iopub.execute_input":"2023-10-25T19:04:58.578923Z","iopub.status.idle":"2023-10-25T19:04:58.635948Z","shell.execute_reply.started":"2023-10-25T19:04:58.578895Z","shell.execute_reply":"2023-10-25T19:04:58.635221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scale features\nscaler = MinMaxScaler()\nscaler_model=scaler.fit(data[['anglez', 'enmo']])\nwith open('standard_scaler_model.pkl', 'wb') as f:\n    pickle.dump(scaler_model, f)\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:04:58.636889Z","iopub.execute_input":"2023-10-25T19:04:58.637205Z","iopub.status.idle":"2023-10-25T19:04:59.391207Z","shell.execute_reply.started":"2023-10-25T19:04:58.637178Z","shell.execute_reply":"2023-10-25T19:04:59.390157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('standard_scaler_model.pkl', 'rb') as file:\n    scaler_model = pickle.load(file)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:04:59.392844Z","iopub.execute_input":"2023-10-25T19:04:59.394162Z","iopub.status.idle":"2023-10-25T19:04:59.401476Z","shell.execute_reply.started":"2023-10-25T19:04:59.394118Z","shell.execute_reply":"2023-10-25T19:04:59.400066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del data","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:04:59.405904Z","iopub.execute_input":"2023-10-25T19:04:59.406868Z","iopub.status.idle":"2023-10-25T19:05:01.421089Z","shell.execute_reply.started":"2023-10-25T19:04:59.406824Z","shell.execute_reply":"2023-10-25T19:05:01.419772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.422467Z","iopub.execute_input":"2023-10-25T19:05:01.422946Z","iopub.status.idle":"2023-10-25T19:05:01.442718Z","shell.execute_reply.started":"2023-10-25T19:05:01.422906Z","shell.execute_reply":"2023-10-25T19:05:01.441145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaled_data=scaler_model.transform(data[['anglez', 'enmo']])\n \n# # print scaled features\n# print(scaled_data)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.448124Z","iopub.execute_input":"2023-10-25T19:05:01.448504Z","iopub.status.idle":"2023-10-25T19:05:01.457506Z","shell.execute_reply.started":"2023-10-25T19:05:01.448475Z","shell.execute_reply":"2023-10-25T19:05:01.456433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some functions","metadata":{}},{"cell_type":"code","source":"def get_file_list_from_dir(datadir):\n    all_files = os.listdir(os.path.abspath(datadir))\n    data_files = list(filter(lambda file: file.endswith('.npy'), all_files))\n    return data_files\nfrom random import shuffle\n\ndef randomize_files(file_list):\n    shuffle(file_list)\nfrom math import floor\n\ndef get_training_and_testing_sets(file_list):\n    split = 0.7\n    split_index = floor(len(file_list) * split)\n    training = file_list[:split_index]\n    testing = file_list[split_index:]\n    return training, testing\ndef concat_files(datadir,file_list):\n    data_files=datadir + pd.Series(file_list)#.astype(str)\n    data_files=data_files.tolist()\n#     print(type(data_files))\n    labels_files=[sub.replace('.npy', '.txt') for sub in data_files]\n#     print(type(labels_files))\n    files_to_move=data_files+labels_files\n#     files_to_move=data_files.tolist().extend(labels_files)\n#     print(\"fisiere concatenate\",len(files_to_move[0]))\n#     print(\"lungime fisiere concatenate\",len(files_to_move))\n    return files_to_move\ndef move_files(datadir):\n    print(datadir)\n    file_list=get_file_list_from_dir(datadir)\n    shuffle(file_list)\n    training, testing=get_training_and_testing_sets(file_list)\n    training=concat_files(datadir,training)\n    testing=concat_files(datadir,testing)\n#     print(\"len training\",len(training))\n#     print(\"len testing\",len(testing))\n#     print(training[10])\n#     print(testing[10])\n\n    for f in training:\n#         print(f)\n#         shutil.copy(f, path_to_train)\n#         os.remove(f)\n#         try:\n#             print('what the hell')\n#             os.unlink(f)\n#         except:\n#             print(\"not the case\")\n#         print('what the hell !!!!!!')\n        shutil.move(f, path_to_train)\n    for f in testing:\n#         print(f)\n#         try:\n#             print('what the hell')\n#             os.unlink(f)\n#         except:\n#             print(\"not the case\")\n#         shutil.copy(f, path_to_test)\n#         os.remove(f)\n#         os.unlink(f)\n#         print('what the hell !!!!!!')\n        shutil.move(f, path_to_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.459264Z","iopub.execute_input":"2023-10-25T19:05:01.459686Z","iopub.status.idle":"2023-10-25T19:05:01.474208Z","shell.execute_reply.started":"2023-10-25T19:05:01.459625Z","shell.execute_reply":"2023-10-25T19:05:01.472668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# temp=train[train['step'].between(10,100)]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.476000Z","iopub.execute_input":"2023-10-25T19:05:01.476401Z","iopub.status.idle":"2023-10-25T19:05:01.490642Z","shell.execute_reply.started":"2023-10-25T19:05:01.476355Z","shell.execute_reply":"2023-10-25T19:05:01.489642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# len(temp)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.492207Z","iopub.execute_input":"2023-10-25T19:05:01.492948Z","iopub.status.idle":"2023-10-25T19:05:01.505369Z","shell.execute_reply.started":"2023-10-25T19:05:01.492907Z","shell.execute_reply":"2023-10-25T19:05:01.504046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def numpy_and_labels(step,lenght,target,filecode):\n    temp=train[train['step'].between(step-lenght,step+lenght)]\n#     serial=temp['series_id'].unique()[0]\n    if len(temp)==121:\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        #target=3\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        if row.shape !=(3,121):\n            print(\"wrong shape\",row.shape) \n        else:\n            filename=str(str(step)+filecode+serial)\n#             print(\"good shape\",row.shape) \n            np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n            with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n                f.write(str(target))\n            #print(row.shape)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.507021Z","iopub.execute_input":"2023-10-25T19:05:01.507429Z","iopub.status.idle":"2023-10-25T19:05:01.518256Z","shell.execute_reply.started":"2023-10-25T19:05:01.507396Z","shell.execute_reply":"2023-10-25T19:05:01.516669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def make_onset_dataset(train,onset_list):\n#     for event in onset_list:\n#         for step in np.arange(event-45, event+45, 5, dtype=int):\n#             lenght=60\n#             target=3\n#             filecode='onset1'\n#             numpy_and_labels(step,lenght,target,filecode)\n#             target=0\n#             filecode='awake1'\n#             offset=step+165\n#             numpy_and_labels(offset,lenght,target,filecode)\n#             target=1\n#             filecode='awake1'\n#             offset=step-165\n#             numpy_and_labels(offset,lenght,target,filecode)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.520746Z","iopub.execute_input":"2023-10-25T19:05:01.521284Z","iopub.status.idle":"2023-10-25T19:05:01.533553Z","shell.execute_reply.started":"2023-10-25T19:05:01.521240Z","shell.execute_reply":"2023-10-25T19:05:01.532273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file_list=get_file_list_from_dir(datadir)\n# shuffle(file_list)\n# training, testing=get_training_and_testing_sets(file_list)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.534920Z","iopub.execute_input":"2023-10-25T19:05:01.535949Z","iopub.status.idle":"2023-10-25T19:05:01.546423Z","shell.execute_reply.started":"2023-10-25T19:05:01.535911Z","shell.execute_reply":"2023-10-25T19:05:01.545030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tags=''.join(random.choices(string.ascii_lowercase, k=5))\nfilecode='onset1'+tags\nfilecode","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.548140Z","iopub.execute_input":"2023-10-25T19:05:01.548900Z","iopub.status.idle":"2023-10-25T19:05:01.562953Z","shell.execute_reply.started":"2023-10-25T19:05:01.548857Z","shell.execute_reply":"2023-10-25T19:05:01.561634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_onset_dataset(train,onset_list):\n    for event in onset_list:\n        for step in np.arange(event-45, event+45, 5, dtype=int):\n            lenght=60\n            target=3\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='onset1'+tags\n            numpy_and_labels(step,lenght,target,filecode)\n            target=0\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='sleep1'+tags\n            offset=step+165\n            numpy_and_labels(offset,lenght,target,filecode)\n            target=1\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='awake1'+tags\n            offset=step-165\n            numpy_and_labels(offset,lenght,target,filecode)\ndef make_wakeup_list_dataset(train,wakeup_list):\n    for event in wakeup_list:\n        for step in np.arange(event-45, event+45, 5, dtype=int):\n            lenght=60\n            target=2\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='wakeup1'+tags\n            numpy_and_labels(step,lenght,target,filecode)\n            target=1\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='awake'+tags\n            offset=step+165\n            numpy_and_labels(offset,lenght,target,filecode)\n            target=0\n            tags=''.join(random.choices(string.ascii_lowercase, k=5))\n            filecode='sleep'+tags\n            offset=step-165\n            numpy_and_labels(offset,lenght,target,filecode)\ndef make_awake_0_dataset(train):\n    full=train[train['awake']==0]\n    lenght=len(full)\n    maxrange=lenght//121\n#     print(len(full))\n    for window in np.arange(0, maxrange, 121, dtype=int):\n        temp=full.iloc[window:window+121]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=0\n        row=np.array([anglez, enmo,convolve], dtype='object')\n#         print(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_p0_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n        gc.collect\ndef make_awake_1_dataset(train):\n    full=train[train['awake']==1]\n    lenght=len(full)\n    maxrange=lenght//121\n#     print(len(full))\n    for window in np.arange(0, maxrange, 121, dtype=int):\n        temp=full.iloc[window:window+121]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        enmo=temp['enmo'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=1\n        row=np.array([anglez, enmo,convolve], dtype='object')\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_s0_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n        gc.collect\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.564177Z","iopub.execute_input":"2023-10-25T19:05:01.564788Z","iopub.status.idle":"2023-10-25T19:05:01.592485Z","shell.execute_reply.started":"2023-10-25T19:05:01.564757Z","shell.execute_reply":"2023-10-25T19:05:01.591231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.595791Z","iopub.execute_input":"2023-10-25T19:05:01.596866Z","iopub.status.idle":"2023-10-25T19:05:01.653587Z","shell.execute_reply.started":"2023-10-25T19:05:01.596822Z","shell.execute_reply":"2023-10-25T19:05:01.652257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.655147Z","iopub.execute_input":"2023-10-25T19:05:01.655592Z","iopub.status.idle":"2023-10-25T19:05:01.680505Z","shell.execute_reply.started":"2023-10-25T19:05:01.655551Z","shell.execute_reply":"2023-10-25T19:05:01.679313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def masking_selector(x):\n    if (x['event_interval'] == 3):\n        return 3\n    elif (x['event_interval'] == 2):\n        return 2\n    elif (x['awake'] ==0):\n        return 0\n    elif (x['awake'] == 1):\n        return x['awake']\n    elif (x['awake'] >100 ):\n        return 999\n# df1 = df1.assign(events=df2.apply(masking_selector, axis=1))","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.682031Z","iopub.execute_input":"2023-10-25T19:05:01.682383Z","iopub.status.idle":"2023-10-25T19:05:01.689274Z","shell.execute_reply.started":"2023-10-25T19:05:01.682354Z","shell.execute_reply":"2023-10-25T19:05:01.687970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_series(series):\n    train_series = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=',series)])\n    with open('standard_scaler_model.pkl', 'rb') as file:\n        scaler_model = pickle.load(file)\n    scaled_data=scaler_model.transform(train_series[['anglez','enmo']])\n    train_series['anglez']= scaled_data[:,0]\n    train_series['enmo']=scaled_data[:,1]\n    train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\").query('series_id == @series')\n    \n    train_events = train_events.dropna()\n    train_events[\"step\"]  = train_events[\"step\"].astype(\"int\")\n    train_events[\"event_interval\"]=train_events[\"event\"].replace({\"onset\":2,\"wakeup\":3})\n\n    #train_events[\"event_interval\"] = pd.to_numeric(train_events[\"event_interval\"], errors='coerce').fillna(0, inplace=True)\n#     train_events[\"event_interval\"]=train_events[\"event_interval\"].fillna(0, inplace=True)\n    train_events[\"awake\"] = train_events[\"event\"].replace({\"onset\":1,\"wakeup\":0})\n    \n    train = pd.merge(train_series, train_events[['step','awake',\"event_interval\"]], on='step', how='left')\n\n    train[\"awake\"] = train[\"awake\"].bfill(axis ='rows')\n    train['awake'] = train['awake'].fillna(1) # awake\n    train[\"awake\"] = train[\"awake\"].astype(\"int\")\n    train[\"event_interval\"] = pd.to_numeric(train[\"event_interval\"], errors='coerce').fillna(0)\n    train[\"event_interval\"] = train[\"event_interval\"].astype(\"int\")\n    onset_list=train.index[(train['event_interval'] ==2)].tolist()\n    wakeup_list=train.index[train['event_interval'] ==3 ].tolist()\n    train['sums']=0\n    train = train.assign(sums=train.apply(masking_selector, axis=1))\n    train[\"awake\"]=train['sums']\n#     train_events[\"event_interval\"] = pd.to_numeric(train_events[\"event_interval\"], errors='coerce').fillna(0)\n#     train_events[\"event_interval\"]=train_events[\"event_interval\"].fillna(0)\n#     train_events[\"event_interval\"]=train_events[\"event_interval\"].astype(\"int\")\n#     train.loc[:,[\"event_interval\"]]=train[\"event_interval\"]+train[\"awake\"]\n\n    train[\"event_interval\"]=train[\"awake\"]\n    for row_index in onset_list:\n        train.iloc[row_index:row_index+120, train.columns.get_loc('event_interval')] = 2\n    for row_index in wakeup_list:\n        train.iloc[row_index:row_index+120, train.columns.get_loc('event_interval')] =3\n    #avoid overlapping windows\n    for row_index in onset_list:\n        train.iloc[row_index:row_index-120, train.columns.get_loc('event_interval')] = 999\n    for row_index in wakeup_list:\n        train.iloc[row_index:row_index-120, train.columns.get_loc('event_interval')] = 999\n    train[\"awake\"]=train[\"event_interval\"]\n    train.drop('event_interval', axis=1, inplace=True)\n    train['convolve']=signal.fftconvolve(train[\"anglez\"].values,train[\"enmo\"].values,  mode='same')\n\n    # final section:\n    # train_events.groupby('series_id').tail(1)[\"event\"].unique()\n    # Result: the last event is always a \"wakeup\"\n    return(train , onset_list,wakeup_list)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.691396Z","iopub.execute_input":"2023-10-25T19:05:01.691887Z","iopub.status.idle":"2023-10-25T19:05:01.714210Z","shell.execute_reply.started":"2023-10-25T19:05:01.691846Z","shell.execute_reply":"2023-10-25T19:05:01.712980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_real_series(path_to_real):\n    real_series = pd.read_parquet(path_to_real)\n    return real_series","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.715748Z","iopub.execute_input":"2023-10-25T19:05:01.716200Z","iopub.status.idle":"2023-10-25T19:05:01.732208Z","shell.execute_reply.started":"2023-10-25T19:05:01.716159Z","shell.execute_reply":"2023-10-25T19:05:01.730714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"get the files","metadata":{}},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.733989Z","iopub.execute_input":"2023-10-25T19:05:01.734367Z","iopub.status.idle":"2023-10-25T19:05:01.765423Z","shell.execute_reply.started":"2023-10-25T19:05:01.734335Z","shell.execute_reply":"2023-10-25T19:05:01.764164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a quick look\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.766937Z","iopub.execute_input":"2023-10-25T19:05:01.767834Z","iopub.status.idle":"2023-10-25T19:05:01.785106Z","shell.execute_reply.started":"2023-10-25T19:05:01.767799Z","shell.execute_reply":"2023-10-25T19:05:01.783763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that all of the series start off in the `awake` state (*i.e.* the very first `event` is the `onset` of sleep):","metadata":{}},{"cell_type":"code","source":"train_events.groupby('series_id').head(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.786849Z","iopub.execute_input":"2023-10-25T19:05:01.787188Z","iopub.status.idle":"2023-10-25T19:05:01.805601Z","shell.execute_reply.started":"2023-10-25T19:05:01.787158Z","shell.execute_reply":"2023-10-25T19:05:01.804308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that all of the series end in the `awake` state (*i.e.* the very last `event` is a `wakeup`):","metadata":{}},{"cell_type":"code","source":"train_events.groupby('series_id').tail(1)[\"event\"].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.807081Z","iopub.execute_input":"2023-10-25T19:05:01.807556Z","iopub.status.idle":"2023-10-25T19:05:01.820196Z","shell.execute_reply.started":"2023-10-25T19:05:01.807517Z","shell.execute_reply":"2023-10-25T19:05:01.819261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"How many series have `NaN` values?","metadata":{}},{"cell_type":"code","source":"series_has_NaN = train_events.groupby('series_id')['step'].apply(lambda x: x.isnull().any())\nseries_has_NaN.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.830894Z","iopub.execute_input":"2023-10-25T19:05:01.832111Z","iopub.status.idle":"2023-10-25T19:05:01.889249Z","shell.execute_reply.started":"2023-10-25T19:05:01.832058Z","shell.execute_reply":"2023-10-25T19:05:01.887986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_has_NaN","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.891052Z","iopub.execute_input":"2023-10-25T19:05:01.891410Z","iopub.status.idle":"2023-10-25T19:05:01.900844Z","shell.execute_reply.started":"2023-10-25T19:05:01.891381Z","shell.execute_reply":"2023-10-25T19:05:01.899620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see the following 37 series have no `NaN` values whatsoever:","metadata":{}},{"cell_type":"code","source":"no_NaN_series = series_has_NaN[~series_has_NaN].index.tolist()\n# no_NaN_series","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.902621Z","iopub.execute_input":"2023-10-25T19:05:01.903404Z","iopub.status.idle":"2023-10-25T19:05:01.913477Z","shell.execute_reply.started":"2023-10-25T19:05:01.903362Z","shell.execute_reply":"2023-10-25T19:05:01.912488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NaN_series=series_has_NaN[series_has_NaN].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.915116Z","iopub.execute_input":"2023-10-25T19:05:01.915869Z","iopub.status.idle":"2023-10-25T19:05:01.928321Z","shell.execute_reply.started":"2023-10-25T19:05:01.915826Z","shell.execute_reply":"2023-10-25T19:05:01.926940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#NaN_series","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.930445Z","iopub.execute_input":"2023-10-25T19:05:01.930900Z","iopub.status.idle":"2023-10-25T19:05:01.942803Z","shell.execute_reply.started":"2023-10-25T19:05:01.930868Z","shell.execute_reply":"2023-10-25T19:05:01.941568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# also drop these two \"truncated\" events series seen in EDA:\nno_NaN_series.remove('31011ade7c0a') # incomplete events data\nno_NaN_series.remove('a596ad0b82aa') # incomplete events data","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.944277Z","iopub.execute_input":"2023-10-25T19:05:01.944628Z","iopub.status.idle":"2023-10-25T19:05:01.956365Z","shell.execute_reply.started":"2023-10-25T19:05:01.944599Z","shell.execute_reply":"2023-10-25T19:05:01.954994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(series_has_NaN)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.958020Z","iopub.execute_input":"2023-10-25T19:05:01.958839Z","iopub.status.idle":"2023-10-25T19:05:01.970308Z","shell.execute_reply.started":"2023-10-25T19:05:01.958792Z","shell.execute_reply":"2023-10-25T19:05:01.969007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using remove() method to remove common elements\nfor i in series_has_NaN[:]:\n    if (i in no_NaN_series):\n        series_has_NaN.remove(i)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.972071Z","iopub.execute_input":"2023-10-25T19:05:01.972539Z","iopub.status.idle":"2023-10-25T19:05:01.983814Z","shell.execute_reply.started":"2023-10-25T19:05:01.972499Z","shell.execute_reply":"2023-10-25T19:05:01.982456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#len(series_has_NaN)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:01.985257Z","iopub.execute_input":"2023-10-25T19:05:01.986284Z","iopub.status.idle":"2023-10-25T19:05:02.001749Z","shell.execute_reply.started":"2023-10-25T19:05:01.986251Z","shell.execute_reply":"2023-10-25T19:05:02.000105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#no_NaN_series","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:02.003345Z","iopub.execute_input":"2023-10-25T19:05:02.003783Z","iopub.status.idle":"2023-10-25T19:05:02.016167Z","shell.execute_reply.started":"2023-10-25T19:05:02.003745Z","shell.execute_reply":"2023-10-25T19:05:02.014616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"test for one series","metadata":{}},{"cell_type":"code","source":"train , onset_list,wakeup_list = get_train_series('08db4255286f')","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:02.017800Z","iopub.execute_input":"2023-10-25T19:05:02.018357Z","iopub.status.idle":"2023-10-25T19:05:21.747770Z","shell.execute_reply.started":"2023-10-25T19:05:02.018316Z","shell.execute_reply":"2023-10-25T19:05:21.746591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.749213Z","iopub.execute_input":"2023-10-25T19:05:21.749669Z","iopub.status.idle":"2023-10-25T19:05:21.766819Z","shell.execute_reply.started":"2023-10-25T19:05:21.749627Z","shell.execute_reply":"2023-10-25T19:05:21.765353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.awake.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.768433Z","iopub.execute_input":"2023-10-25T19:05:21.768852Z","iopub.status.idle":"2023-10-25T19:05:21.789302Z","shell.execute_reply.started":"2023-10-25T19:05:21.768816Z","shell.execute_reply":"2023-10-25T19:05:21.787798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# onset_list","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.790789Z","iopub.execute_input":"2023-10-25T19:05:21.791844Z","iopub.status.idle":"2023-10-25T19:05:21.801436Z","shell.execute_reply.started":"2023-10-25T19:05:21.791809Z","shell.execute_reply":"2023-10-25T19:05:21.800256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# wakeup_list","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.803128Z","iopub.execute_input":"2023-10-25T19:05:21.803604Z","iopub.status.idle":"2023-10-25T19:05:21.814987Z","shell.execute_reply.started":"2023-10-25T19:05:21.803561Z","shell.execute_reply":"2023-10-25T19:05:21.813576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step=1","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.816445Z","iopub.execute_input":"2023-10-25T19:05:21.816842Z","iopub.status.idle":"2023-10-25T19:05:21.830439Z","shell.execute_reply.started":"2023-10-25T19:05:21.816810Z","shell.execute_reply":"2023-10-25T19:05:21.828969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=train[train['step']==step]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.832198Z","iopub.execute_input":"2023-10-25T19:05:21.832716Z","iopub.status.idle":"2023-10-25T19:05:21.845517Z","shell.execute_reply.started":"2023-10-25T19:05:21.832646Z","shell.execute_reply":"2023-10-25T19:05:21.844220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp=train[train['step'].between(99, 101)]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.847537Z","iopub.execute_input":"2023-10-25T19:05:21.848179Z","iopub.status.idle":"2023-10-25T19:05:21.858674Z","shell.execute_reply.started":"2023-10-25T19:05:21.848134Z","shell.execute_reply":"2023-10-25T19:05:21.857746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.860240Z","iopub.execute_input":"2023-10-25T19:05:21.860904Z","iopub.status.idle":"2023-10-25T19:05:21.879131Z","shell.execute_reply.started":"2023-10-25T19:05:21.860872Z","shell.execute_reply":"2023-10-25T19:05:21.877789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test and validation","metadata":{}},{"cell_type":"code","source":"for series in no_NaN_series:\n# for series in no_NaN_series:\n    serial=series\n    train , onset_list,wakeup_list=get_train_series(series)\n    make_onset_dataset(train,onset_list)\n    make_wakeup_list_dataset(train,wakeup_list)\n    make_awake_0_dataset(train)\n    make_awake_1_dataset(train)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:05:21.880745Z","iopub.execute_input":"2023-10-25T19:05:21.881419Z","iopub.status.idle":"2023-10-25T19:18:18.586355Z","shell.execute_reply.started":"2023-10-25T19:05:21.881385Z","shell.execute_reply":"2023-10-25T19:18:18.583991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.getcwd()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:18:18.589101Z","iopub.execute_input":"2023-10-25T19:18:18.589636Z","iopub.status.idle":"2023-10-25T19:18:18.597116Z","shell.execute_reply.started":"2023-10-25T19:18:18.589585Z","shell.execute_reply":"2023-10-25T19:18:18.595604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os.unlink('/kaggle/working/dataset/signals120/complete/27398808db4255286f.npy') ","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:18:18.599167Z","iopub.execute_input":"2023-10-25T19:18:18.599870Z","iopub.status.idle":"2023-10-25T19:18:18.613193Z","shell.execute_reply.started":"2023-10-25T19:18:18.599817Z","shell.execute_reply":"2023-10-25T19:18:18.611720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"move_files(datadir)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:18:18.615287Z","iopub.execute_input":"2023-10-25T19:18:18.615941Z","iopub.status.idle":"2023-10-25T19:18:32.913860Z","shell.execute_reply.started":"2023-10-25T19:18:18.615897Z","shell.execute_reply":"2023-10-25T19:18:32.912475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extended dataset","metadata":{}},{"cell_type":"code","source":"def make_onset_nan_dataset(train,onset_list):\n    for step in onset_list:\n        temp=train[train['step'].between(step,step+120)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=2\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A2_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n             f.write(str(target))\n        temp=train[train['step'].between(step+120,step+240)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=2\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A22_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\ndef make_nonset_nan_dataset(train,onset_list):\n    for step in onset_list:\n        temp=train[train['step'].between(step-120,step)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=1\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A1_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n\ndef make_wakeup_nan_dataset(train,wakeup_list):\n    for step in wakeup_list:\n        temp=train[train['step'].between(step,step+120)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=3\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A3_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n        temp=train[train['step'].between(step+120,step+240)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=1\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A31_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n\ndef make_nwakeup_nan_dataset(train,wakeup_list):\n    for step in wakeup_list:\n        temp=train[train['step'].between(step-120,step)]\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        target=0\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        row=np.nan_to_num(row)\n#         print(row.shape)\n        tags=''.join(random.choices(string.ascii_lowercase, k=5))\n        filename=str(str(step)+\"_A0_\"+serial+tags)\n        np.save(f'/kaggle/working/dataset/signals120/complete/{filename}', row)\n        with open(f'/kaggle/working/dataset/signals120/complete/{filename}.txt', 'w', encoding='utf-8') as f:\n            f.write(str(target))\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:18:32.915355Z","iopub.execute_input":"2023-10-25T19:18:32.915742Z","iopub.status.idle":"2023-10-25T19:18:32.951393Z","shell.execute_reply.started":"2023-10-25T19:18:32.915711Z","shell.execute_reply":"2023-10-25T19:18:32.949742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series in NaN_series:\n    try:\n        train , onset_list,wakeup_list=get_train_series(series)\n        make_onset_dataset(train,onset_list)\n        make_wakeup_list_dataset(train,wakeup_list)\n        make_onset_nan_dataset(train,onset_list)\n        make_wakeup_nan_dataset(train,wakeup_list)\n        make_nwakeup_nan_dataset(train,wakeup_list)\n        make_nonset_nan_dataset(train,onset_list)\n    except:\n        print('ignored series',series)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T19:18:32.952969Z","iopub.execute_input":"2023-10-25T19:18:32.953350Z","iopub.status.idle":"2023-10-25T20:59:12.343544Z","shell.execute_reply.started":"2023-10-25T19:18:32.953316Z","shell.execute_reply":"2023-10-25T20:59:12.340305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"move_files(datadir)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T20:59:12.347402Z","iopub.execute_input":"2023-10-25T20:59:12.348077Z","iopub.status.idle":"2023-10-25T21:01:25.437110Z","shell.execute_reply.started":"2023-10-25T20:59:12.348024Z","shell.execute_reply":"2023-10-25T21:01:25.435743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check the files were moved","metadata":{}},{"cell_type":"code","source":"file_list=get_file_list_from_dir(datadir)\nfile_list","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.438887Z","iopub.execute_input":"2023-10-25T21:01:25.439310Z","iopub.status.idle":"2023-10-25T21:01:25.464762Z","shell.execute_reply.started":"2023-10-25T21:01:25.439275Z","shell.execute_reply":"2023-10-25T21:01:25.463748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import sys\n\n# These are the usual ipython objects, including this one you are creating\nipython_vars = [\"In\", \"Out\", \"exit\", \"quit\", \"get_ipython\", \"ipython_vars\"]\n\n# Get a sorted list of the objects and their sizes\nmem = {\n    key: value\n    for key, value in sorted(\n        [\n            (x, sys.getsizeof(globals().get(x)))\n            for x in dir()\n            if not x.startswith(\"_\") and x not in sys.modules and x not in ipython_vars\n        ],\n        key=lambda x: x[1],\n        reverse=True,\n    )\n}\nmem","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.466085Z","iopub.execute_input":"2023-10-25T21:01:25.467127Z","iopub.status.idle":"2023-10-25T21:01:25.807516Z","shell.execute_reply.started":"2023-10-25T21:01:25.467091Z","shell.execute_reply":"2023-10-25T21:01:25.806592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    del train,train_events,file_list,series_has_NaN\n    gc.collect\nexcept:\n    print(a)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.809016Z","iopub.execute_input":"2023-10-25T21:01:25.809727Z","iopub.status.idle":"2023-10-25T21:01:25.836242Z","shell.execute_reply.started":"2023-10-25T21:01:25.809676Z","shell.execute_reply":"2023-10-25T21:01:25.834923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Real data to predict","metadata":{}},{"cell_type":"code","source":"# def make_real_test(real_test):\n#     lenght=len(real_test)\n#     maxrange=lenght//120\n#     print(maxrange)\n#     for window in np.arange(0, maxrange, 120, dtype=int):\n#         temp=real_test.iloc[window:window+121]\n#         serial=temp['series_id'].unique()[0]\n#         enmo=temp['enmo'].to_numpy()\n#         anglez=temp['anglez'].to_numpy()\n#         row=np.array([step, anglez, enmo], dtype='object')\n#         filename=str(str(window)+serial)\n#     #   print(\"fisier\",filename,\"nume\")\n#         np.save(f'/kaggle/working/dataset/signals120/real/{filename}', row)\n#         gc.collect\n#         print(temp.shape)\n# #         print(temp)\ndef make_real_test(real_test):\n    lenght=len(real_test)\n    maxrange=lenght//121\n    print(maxrange)\n    real_test['convolve']=0\n    for window in np.arange(0, maxrange, 121, dtype=int):\n        \n        temp=real_test.iloc[window:window+121]\n        scaled_data=scaler_model.transform(temp[['anglez','enmo']])\n        temp.loc[:,'anglez']= scaled_data[:,0]\n        temp.loc[:,'enmo']=scaled_data[:,1]\n        temp.loc[:,'convolve']=signal.fftconvolve(temp[\"anglez\"].values,temp[\"enmo\"].values,  mode='same')\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        filename=str(str(window)+serial)\n    #   print(\"fisier\",filename,\"nume\")\n        np.save(f'/kaggle/working/dataset/signals120/real/{filename}', row)\n        gc.collect\n        print(temp.shape, row.shape)\n#         print(temp)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.837736Z","iopub.execute_input":"2023-10-25T21:01:25.838164Z","iopub.status.idle":"2023-10-25T21:01:25.854779Z","shell.execute_reply.started":"2023-10-25T21:01:25.838131Z","shell.execute_reply":"2023-10-25T21:01:25.853565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_test=get_real_series(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.856408Z","iopub.execute_input":"2023-10-25T21:01:25.857081Z","iopub.status.idle":"2023-10-25T21:01:25.888249Z","shell.execute_reply.started":"2023-10-25T21:01:25.857039Z","shell.execute_reply":"2023-10-25T21:01:25.886978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_test_series=real_test['series_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.889929Z","iopub.execute_input":"2023-10-25T21:01:25.890445Z","iopub.status.idle":"2023-10-25T21:01:25.897136Z","shell.execute_reply.started":"2023-10-25T21:01:25.890400Z","shell.execute_reply":"2023-10-25T21:01:25.896026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_test_series","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.898635Z","iopub.execute_input":"2023-10-25T21:01:25.899343Z","iopub.status.idle":"2023-10-25T21:01:25.918113Z","shell.execute_reply.started":"2023-10-25T21:01:25.899293Z","shell.execute_reply":"2023-10-25T21:01:25.915696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xxx=real_test[real_test[\"series_id\"]=='038441c925bb']","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.920878Z","iopub.execute_input":"2023-10-25T21:01:25.922526Z","iopub.status.idle":"2023-10-25T21:01:25.941840Z","shell.execute_reply.started":"2023-10-25T21:01:25.922462Z","shell.execute_reply":"2023-10-25T21:01:25.940173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xxx","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.943748Z","iopub.execute_input":"2023-10-25T21:01:25.945164Z","iopub.status.idle":"2023-10-25T21:01:25.980272Z","shell.execute_reply.started":"2023-10-25T21:01:25.945101Z","shell.execute_reply":"2023-10-25T21:01:25.978595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(xxx)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:25.981840Z","iopub.execute_input":"2023-10-25T21:01:25.982215Z","iopub.status.idle":"2023-10-25T21:01:25.998335Z","shell.execute_reply.started":"2023-10-25T21:01:25.982180Z","shell.execute_reply":"2023-10-25T21:01:25.997117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series in real_test_series:\n    real_test=pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\", filters=[('series_id','=',series)])\n    make_real_test(real_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:26.000351Z","iopub.execute_input":"2023-10-25T21:01:26.000740Z","iopub.status.idle":"2023-10-25T21:01:26.054641Z","shell.execute_reply.started":"2023-10-25T21:01:26.000710Z","shell.execute_reply":"2023-10-25T21:01:26.053749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read / load data","metadata":{}},{"cell_type":"code","source":"1+1","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:26.056754Z","iopub.execute_input":"2023-10-25T21:01:26.057619Z","iopub.status.idle":"2023-10-25T21:01:26.067683Z","shell.execute_reply.started":"2023-10-25T21:01:26.057573Z","shell.execute_reply":"2023-10-25T21:01:26.064825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# path_to_train=\"/kaggle/input/zzzs-make-small-starter-dataset-npy/dataset/signals120/train\"\n# path_to_test=\"/kaggle/input/zzzs-make-small-starter-dataset-npy/dataset/signals120/val\"\n# #/kaggle/working/dataset/signals120/train","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:26.069198Z","iopub.execute_input":"2023-10-25T21:01:26.070179Z","iopub.status.idle":"2023-10-25T21:01:26.080378Z","shell.execute_reply.started":"2023-10-25T21:01:26.070044Z","shell.execute_reply":"2023-10-25T21:01:26.079098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_numpy_labels_from_dir(datadir):\n    all_files = os.listdir(os.path.abspath(datadir))\n    data_files = list(filter(lambda file: file.endswith('.npy'), all_files))\n    label_files = list(filter(lambda file: file.endswith('.txt'), all_files))\n    data_files_path= [datadir+\"/\"+ s for s in data_files]\n    label_files_path= [datadir+\"/\"+ s for s in label_files]\n    data_files_path.sort()\n    label_files_path.sort()\n    return data_files_path , label_files_path","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:26.082375Z","iopub.execute_input":"2023-10-25T21:01:26.082776Z","iopub.status.idle":"2023-10-25T21:01:26.097478Z","shell.execute_reply.started":"2023-10-25T21:01:26.082745Z","shell.execute_reply":"2023-10-25T21:01:26.095956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files,labels_train=get_numpy_labels_from_dir(path_to_train)\nvalidation_files,labels_test=get_numpy_labels_from_dir(path_to_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:26.099156Z","iopub.execute_input":"2023-10-25T21:01:26.099561Z","iopub.status.idle":"2023-10-25T21:01:28.745200Z","shell.execute_reply.started":"2023-10-25T21:01:26.099528Z","shell.execute_reply":"2023-10-25T21:01:28.744217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_files=train_files[0:100]\n# labels_train=labels_train[0:100]\n# validation_files=validation_files[0:100]\n# labels_test=labels_test[0:100]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.746591Z","iopub.execute_input":"2023-10-25T21:01:28.747282Z","iopub.status.idle":"2023-10-25T21:01:28.752405Z","shell.execute_reply.started":"2023-10-25T21:01:28.747247Z","shell.execute_reply":"2023-10-25T21:01:28.750902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_files[0:6]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.754230Z","iopub.execute_input":"2023-10-25T21:01:28.754606Z","iopub.status.idle":"2023-10-25T21:01:28.817642Z","shell.execute_reply.started":"2023-10-25T21:01:28.754575Z","shell.execute_reply":"2023-10-25T21:01:28.816423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_test[0:6]","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.819727Z","iopub.execute_input":"2023-10-25T21:01:28.820529Z","iopub.status.idle":"2023-10-25T21:01:28.834241Z","shell.execute_reply.started":"2023-10-25T21:01:28.820491Z","shell.execute_reply":"2023-10-25T21:01:28.833166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# '1065003664fe9233f9'=='1065003664fe9233f9'","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.836985Z","iopub.execute_input":"2023-10-25T21:01:28.837375Z","iopub.status.idle":"2023-10-25T21:01:28.846625Z","shell.execute_reply.started":"2023-10-25T21:01:28.837345Z","shell.execute_reply":"2023-10-25T21:01:28.845074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load data","metadata":{}},{"cell_type":"code","source":"def load_data(path_list):\n    np_list = []      \n    for path in path_list:\n        np_list.append(np.load(path, allow_pickle=True))      \n    return np_list","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.848186Z","iopub.execute_input":"2023-10-25T21:01:28.848565Z","iopub.status.idle":"2023-10-25T21:01:28.859238Z","shell.execute_reply.started":"2023-10-25T21:01:28.848525Z","shell.execute_reply":"2023-10-25T21:01:28.857750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(path_list):\n    np_list = []\n    error_list=[]\n    for path in path_list:\n        elem=np.load(path, allow_pickle=True)\n        if elem.shape==(3, 121):\n            np_list.append(elem)\n        else:\n            print(path)\n            print(elem.shape)\n            error_list.append(path)\n    labels_files=[sub.replace('.npy', '.txt') for sub in error_list]\n    return np_list , labels_files","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.861508Z","iopub.execute_input":"2023-10-25T21:01:28.861954Z","iopub.status.idle":"2023-10-25T21:01:28.874226Z","shell.execute_reply.started":"2023-10-25T21:01:28.861905Z","shell.execute_reply":"2023-10-25T21:01:28.872989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def load_data_and_labels(path_list,text_path):\n#     np_list = []\n#     error_list=[]\n#     labels = []\n#     for path in path_list:\n#         elem=np.load(path, allow_pickle=True)\n#         if elem.shape==(3, 121):\n#             np_list.append(elem)\n#             text_file = open(path, 'r', encoding=\"utf8\", errors='ignore')\n#             contents = text_file.read()\n#             print (contents)\n# #             contents = f.read()\n#             labels.append(text_file.read()) \n#             text_file.close()\n#         else:\n#             print(path)\n#             print(elem.shape)\n#             error_list.append(path)\n#             text_file = open(path, 'r')\n#             print(\"eticheta este:\",text_file.read())\n#             text_file.close()\n    \n#     return np_list,labels","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.875999Z","iopub.execute_input":"2023-10-25T21:01:28.876481Z","iopub.status.idle":"2023-10-25T21:01:28.890558Z","shell.execute_reply.started":"2023-10-25T21:01:28.876437Z","shell.execute_reply":"2023-10-25T21:01:28.889389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#for multiclass \ndef get_labels(file_list, labels_files):\n    labels = [] #-->my list variable\n    for file_path in file_list:\n        if file_path not in labels_files:\n            text_file = open(file_path, 'r')\n            labels.append(text_file.read()) \n            text_file.close()\n    labels = [int(i) for i in labels]\n    return labels","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.892574Z","iopub.execute_input":"2023-10-25T21:01:28.893094Z","iopub.status.idle":"2023-10-25T21:01:28.908378Z","shell.execute_reply.started":"2023-10-25T21:01:28.893047Z","shell.execute_reply":"2023-10-25T21:01:28.907465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Numpy dataset ready","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\nlabel_as_binary = LabelBinarizer()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.909968Z","iopub.execute_input":"2023-10-25T21:01:28.910905Z","iopub.status.idle":"2023-10-25T21:01:28.923920Z","shell.execute_reply.started":"2023-10-25T21:01:28.910858Z","shell.execute_reply":"2023-10-25T21:01:28.922427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,labels_files=load_data(train_files)\nX_train= np.asarray(X_train).astype('float32')\n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:01:28.926387Z","iopub.execute_input":"2023-10-25T21:01:28.926958Z","iopub.status.idle":"2023-10-25T21:07:52.602404Z","shell.execute_reply.started":"2023-10-25T21:01:28.926909Z","shell.execute_reply":"2023-10-25T21:07:52.601076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=get_labels(labels_train,labels_files)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:07:52.604361Z","iopub.execute_input":"2023-10-25T21:07:52.608153Z","iopub.status.idle":"2023-10-25T21:10:52.138557Z","shell.execute_reply.started":"2023-10-25T21:07:52.608096Z","shell.execute_reply":"2023-10-25T21:10:52.136993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.DataFrame(y_train)\ndf.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:10:52.140206Z","iopub.execute_input":"2023-10-25T21:10:52.140754Z","iopub.status.idle":"2023-10-25T21:10:52.353803Z","shell.execute_reply.started":"2023-10-25T21:10:52.140706Z","shell.execute_reply":"2023-10-25T21:10:52.352611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train=get_labels(labels_train,labels_files)\n\ntrain_y_bin = label_as_binary.fit_transform(y_train)\nX_val,labels_files= load_data(validation_files)\nX_val=np.asarray(X_val).astype('float32')\ny_val=get_labels(labels_test,labels_files)\nval_y_bin= label_as_binary.fit_transform(y_val)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:10:52.355308Z","iopub.execute_input":"2023-10-25T21:10:52.355756Z","iopub.status.idle":"2023-10-25T21:13:54.627833Z","shell.execute_reply.started":"2023-10-25T21:10:52.355724Z","shell.execute_reply":"2023-10-25T21:13:54.626541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.DataFrame(y_train)\ndf.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:13:54.644587Z","iopub.execute_input":"2023-10-25T21:13:54.645196Z","iopub.status.idle":"2023-10-25T21:13:54.851150Z","shell.execute_reply.started":"2023-10-25T21:13:54.645161Z","shell.execute_reply":"2023-10-25T21:13:54.850297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers import *  \nfrom keras.models import Model  \n","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:13:54.852256Z","iopub.execute_input":"2023-10-25T21:13:54.853165Z","iopub.status.idle":"2023-10-25T21:14:06.332158Z","shell.execute_reply.started":"2023-10-25T21:13:54.853131Z","shell.execute_reply":"2023-10-25T21:14:06.331044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=128\ntrain_dataset = tf.data.Dataset.from_tensor_slices((X_train, train_y_bin))\n#shuffle data https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle\ntrain_dataset = train_dataset.shuffle(train_dataset.cardinality())\ntrain_dataset = train_dataset.batch(batch_size)\ntest_dataset = tf.data.Dataset.from_tensor_slices((X_val, val_y_bin))\ntest_dataset=test_dataset.batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:06.335626Z","iopub.execute_input":"2023-10-25T21:14:06.336769Z","iopub.status.idle":"2023-10-25T21:14:07.008083Z","shell.execute_reply.started":"2023-10-25T21:14:06.336728Z","shell.execute_reply":"2023-10-25T21:14:07.006951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.cardinality().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.009864Z","iopub.execute_input":"2023-10-25T21:14:07.010563Z","iopub.status.idle":"2023-10-25T21:14:07.017645Z","shell.execute_reply.started":"2023-10-25T21:14:07.010529Z","shell.execute_reply":"2023-10-25T21:14:07.016133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X_train shape:\", X_train.shape)\nprint(X_train.shape[0], \"train samples\")\nprint(X_val.shape[0], \"test samples\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.019424Z","iopub.execute_input":"2023-10-25T21:14:07.020384Z","iopub.status.idle":"2023-10-25T21:14:07.034866Z","shell.execute_reply.started":"2023-10-25T21:14:07.020349Z","shell.execute_reply":"2023-10-25T21:14:07.033561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape=(3,121)\nnum_classes=4","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.036887Z","iopub.execute_input":"2023-10-25T21:14:07.037257Z","iopub.status.idle":"2023-10-25T21:14:07.049047Z","shell.execute_reply.started":"2023-10-25T21:14:07.037227Z","shell.execute_reply":"2023-10-25T21:14:07.047941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class_weight = {0: 2.,\n#                 1: 1.,\n#                 2: 0.6,\n#                 3: 0.3}\n# class_weight = {0: 0.5,\n#                 1: 0.5,\n#                 2: 1,\n#                 3: 1.2}\nclass_weight = {0: 1.2,\n                1: 1.2,\n                2: 0.6,\n                3: 0.6}","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.050308Z","iopub.execute_input":"2023-10-25T21:14:07.050956Z","iopub.status.idle":"2023-10-25T21:14:07.064595Z","shell.execute_reply.started":"2023-10-25T21:14:07.050922Z","shell.execute_reply":"2023-10-25T21:14:07.063032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"help(layers)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.066336Z","iopub.execute_input":"2023-10-25T21:14:07.067023Z","iopub.status.idle":"2023-10-25T21:14:07.082410Z","shell.execute_reply.started":"2023-10-25T21:14:07.066977Z","shell.execute_reply":"2023-10-25T21:14:07.081484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential([\n  keras.Input(shape=input_shape),\n  layers.Dense(1024, activation='relu'),\n  layers.Dropout(.1),\n  layers.Dense(512, activation='relu'),\n  layers.Dropout(.1),\n  layers.Dense(256, activation='relu'),\n  layers.Dropout(.1),\n  layers.Dense(128, activation='relu'),\n  layers.Dense(64, activation='relu'),\n  layers.Dropout(.1),\n  layers.Dense(32, activation='relu'),\n  layers.Flatten(),\n  layers.Dropout(.1),\n  layers.Dense(num_classes, activation=\"softmax\")\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.083944Z","iopub.execute_input":"2023-10-25T21:14:07.084696Z","iopub.status.idle":"2023-10-25T21:14:07.547702Z","shell.execute_reply.started":"2023-10-25T21:14:07.084625Z","shell.execute_reply":"2023-10-25T21:14:07.546340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = keras.Sequential(\n#     [\n#         keras.Input(shape=input_shape),\n\n#         layers.Dense(768, activation=\"relu\"),\n#         layers.Dropout(0.3),\n#         layers.Dense(1024, activation=\"relu\"),\n#         layers.Dropout(0.5),\n#         layers.Dense(256, activation=\"relu\"),\n#         layers.Dropout(0.3),\n# #         layers.Dense(128, activation=\"relu\"),\n# #         layers.Dropout(0.3),\n#         layers.Flatten(),\n#         layers.Dense(num_classes, activation=\"softmax\"),\n#     ]\n# )\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.549547Z","iopub.execute_input":"2023-10-25T21:14:07.549955Z","iopub.status.idle":"2023-10-25T21:14:07.555432Z","shell.execute_reply.started":"2023-10-25T21:14:07.549924Z","shell.execute_reply":"2023-10-25T21:14:07.554159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = keras.Sequential(\n#     [\n#         keras.Input(shape=input_shape),\n\n#         layers.Dense(768, activation=\"relu\"),\n#         layers.Dropout(0.3),\n#         layers.Dense(1024, activation=\"relu\"),\n#         layers.Dropout(0.5),\n#         layers.Dense(256, activation=\"relu\"),\n#         layers.Dropout(0.3),\n# #         layers.Dense(128, activation=\"relu\"),\n# #         layers.Dropout(0.3),\n#         layers.Flatten(),\n#         layers.Dense(num_classes, activation=\"softmax\"),\n#     ]\n# )\n# model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.557419Z","iopub.execute_input":"2023-10-25T21:14:07.557910Z","iopub.status.idle":"2023-10-25T21:14:07.571037Z","shell.execute_reply.started":"2023-10-25T21:14:07.557863Z","shell.execute_reply":"2023-10-25T21:14:07.569727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 256\nepochs = 125\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"categorical_accuracy\"])\n\nhistory = model.fit(train_dataset,validation_data=test_dataset, batch_size=batch_size, epochs=epochs, class_weight=class_weight)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T21:14:07.573034Z","iopub.execute_input":"2023-10-25T21:14:07.573484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history[\"categorical_accuracy\"])\nplt.plot(history.history['val_categorical_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('signal_model.pkl', 'wb') as f:\n    pickle.dump(model, f)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"model_signal.keras\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix,ConfusionMatrixDisplay","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_values = np.concatenate([x for x, y in test_dataset], axis=0)\n\ntest_label = np.concatenate([y for x, y in test_dataset], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_values)\nconfusion_matrix = multilabel_confusion_matrix(test_label, np.rint(y_pred))\n#del confusion_matrix","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, axes = plt.subplots(1, 4, figsize=(25, 5))\naxes = axes.ravel()\nfor i in range(4):\n    disp = ConfusionMatrixDisplay(confusion_matrix[i])\n    disp.plot(ax=axes[i], values_format='.4g')\n    disp.ax_.set_title(f'class {i}')\n    if i<10:\n        disp.ax_.set_xlabel('')\n    if i%5!=0:\n        disp.ax_.set_ylabel('')\n    disp.im_.colorbar.remove()\n\nplt.subplots_adjust(wspace=0.10, hspace=0.1)\nf.colorbar(disp.im_, ax=axes)\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\", filters=[('series_id','=','08db4255286f')])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_real_test(real_test):\n    lenght=len(real_test)\n    maxrange=lenght//121\n#     print(maxrange)\n    for window in np.arange(0, maxrange, 121, dtype=int):\n        real_test['convolve']=0\n        temp=real_test.iloc[window:window+121]\n        scaled_data=scaler_model.transform(temp[['anglez','enmo']])\n        temp.loc[:,'anglez']= scaled_data[:,0]\n        temp.loc[:,'enmo']=scaled_data[:,1]\n        temporar=signal.fftconvolve(temp[\"anglez\"].values,temp[\"enmo\"].values,  mode='same')\n        temp.loc[:,'convolve']=pd.Series(temporar)  \n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        filename=str(str(window)+serial)\n    #   print(\"fisier\",filename,\"nume\")\n        np.save(f'/kaggle/working/dataset/signals120/real/{filename}', row)\n        gc.collect\n#         print(temp.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_real_test(data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_files,labels_real=get_numpy_labels_from_dir(path_to_real)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(path_list):\n    np_list = []\n    error_list=[]\n    for path in path_list:\n        elem=np.load(path, allow_pickle=True)\n        if elem.shape==(3, 121):\n            np_list.append(elem)\n        else:\n            print(path)\n            print(elem.shape)\n            error_list.append(path)\n    labels_files=[sub.replace('.npy', '.txt') for sub in error_list]\n    return np_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_real=load_data(real_files)\nX_real= np.asarray(X_real).astype('float32')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_dataset = tf.data.Dataset.from_tensor_slices((X_train, train_y_bin))\n#shuffle data https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle\nreal_dataset = train_dataset.shuffle(train_dataset.cardinality())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(real_dataset)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1=pd.DataFrame(y_pred)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.loc[:,2]> 0.6","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series=\"1234\"\nmaxValueIndex = df1.idxmax(axis=1)\nscore = df1.max(skipna=False,axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(score)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step_index=range(0,len(score),121)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A very poor model but for a proof of concept it works :)","metadata":{}},{"cell_type":"code","source":"temp_submission=pd.DataFrame()\ntemp_submission['series_id']=str(series)\ntemp_submission['step']=step_index\ntemp_submission['event_id']=maxValueIndex\ntemp_submission['score']=score\ntemp_submission.head(500)\n# temp_submission.loc[temp_submission['event_id'].str.contains(\"youtub\"),'test'] = 'Youtube'\n# temp_submission.loc[temp_submission['event_id'].str.contains(\"face\"),'test'] = 'Facebook'\nreplacements = {\n  r'0': 'sleep',\n  r'1': 'awake',\n  r'2': 'onset',\n  r'3': 'wakeup'\n}\n\ntemp_submission['event'] = temp_submission.event_id.astype(str).replace(replacements, regex=True)\ntemp_submission=temp_submission.loc[(temp_submission.score >0.9)]\ntemp_submission=temp_submission.loc[(temp_submission.event=='onset')|(temp_submission.event ==\"wakeup\")]\n#remove duplicates keep the highes score could be better:)\n#temp_submission.loc[temp_submission.shift() != temp_submission]\ntemp_submission= temp_submission.loc[temp_submission.event_id.shift() != temp_submission.event_id]\n# # df.loc[df.text.str.contains('\\.'), 'text'] = 'other'\nprint(temp_submission)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_submission.tail(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_submission.describe","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1.head(20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp2 = df1[(df1.loc[:,2]> 0.7) | (df1.loc[:,3]> 0.7)]\n#  df_train[(df_train[\"fold\"]==1) | (df_train[\"fold\"]==2)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}