{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n\n\nIn this competition we are given 277 series. However, for 8 of these series we have absolutely no event data (`onset` or `wakeup` timestamps). Indeed, there are only 37 series of events data for which there are no missing events at all, and two of these seem to be truncated. \n\nWithout the corresponding event data it is particularly hard to untangle the sleeping from the awake data contained in the `train_series` data. With this in mind here we create a smaller training dataset consisting of a selection of just 35 of the series. We also add a binary target column called `awake`.\n\n(For plots of all 277 series see the notebook [\"Zzzs: Creating a \"target\" column + viz of all data\"](https://www.kaggle.com/code/carlmcbrideellis/zzzs-creating-a-target-column-viz-of-all-data))\n\n# `Zzzs_train.parquet`\nBinary dataset (`awake = 1, sleeping = 0`,'onset'=6,'wakeup'=7)","metadata":{}},{"cell_type":"code","source":"import os\nimport shutil\nfrom os.path import isfile, join\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom IPython.display import Markdown\nimport gc\nimport numpy as np\nimport pickle\nfrom scipy import signal\nfrom sklearn.preprocessing import MinMaxScaler","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.429248Z","iopub.execute_input":"2023-10-15T15:06:48.429669Z","iopub.status.idle":"2023-10-15T15:06:48.434801Z","shell.execute_reply.started":"2023-10-15T15:06:48.429635Z","shell.execute_reply":"2023-10-15T15:06:48.433727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('standard_scaler_model.pkl', 'rb') as file:\n    scaler_model = pickle.load(file)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read / load data","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_train=\"/kaggle/input/zzzs-make-small-starter-dataset-npy/dataset/signals120/train\"\npath_to_test=\"/kaggle/input/zzzs-make-small-starter-dataset-npy/dataset/signals120/val\"\n#/kaggle/working/dataset/signals120/train","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.439381Z","iopub.execute_input":"2023-10-15T15:06:48.439699Z","iopub.status.idle":"2023-10-15T15:06:48.458079Z","shell.execute_reply.started":"2023-10-15T15:06:48.439673Z","shell.execute_reply":"2023-10-15T15:06:48.456939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_numpy_labels_from_dir(datadir):\n    all_files = os.listdir(os.path.abspath(datadir))\n    data_files = list(filter(lambda file: file.endswith('.npy'), all_files))\n    label_files = list(filter(lambda file: file.endswith('.txt'), all_files))\n    data_files_path= [datadir+\"/\"+ s for s in data_files]\n    label_files_path= [datadir+\"/\"+ s for s in label_files]\n    data_files_path.sort()\n    label_files_path.sort()\n    return data_files_path , label_files_path","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.459915Z","iopub.execute_input":"2023-10-15T15:06:48.460282Z","iopub.status.idle":"2023-10-15T15:06:48.472843Z","shell.execute_reply.started":"2023-10-15T15:06:48.460239Z","shell.execute_reply":"2023-10-15T15:06:48.471734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files,labels_train=get_numpy_labels_from_dir(path_to_train)\nvalidation_files,labels_test=get_numpy_labels_from_dir(path_to_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.473992Z","iopub.execute_input":"2023-10-15T15:06:48.474300Z","iopub.status.idle":"2023-10-15T15:06:48.571947Z","shell.execute_reply.started":"2023-10-15T15:06:48.474275Z","shell.execute_reply":"2023-10-15T15:06:48.570745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files=train_files\nlabels_train=labels_train\nvalidation_files=validation_files\nlabels_test=labels_test","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.574779Z","iopub.execute_input":"2023-10-15T15:06:48.575179Z","iopub.status.idle":"2023-10-15T15:06:48.579370Z","shell.execute_reply.started":"2023-10-15T15:06:48.575147Z","shell.execute_reply":"2023-10-15T15:06:48.578591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_files[0:6]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.580639Z","iopub.execute_input":"2023-10-15T15:06:48.580947Z","iopub.status.idle":"2023-10-15T15:06:48.598415Z","shell.execute_reply.started":"2023-10-15T15:06:48.580920Z","shell.execute_reply":"2023-10-15T15:06:48.597346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"train and test dataset \nmulticlas\nsignal frames with 121 steps and labels [0,1,2,3] sleel, awake, onset, wakeup","metadata":{}},{"cell_type":"code","source":"for path in validation_files[0:6]:\n    values=np.load(path, allow_pickle=True)\n#     print(values.shape)\n#     print(values[0][:])\n    plt.plot(values[0][:])\n    plt.show()\n    plt.plot(values[1][:])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T20:02:36.202036Z","iopub.execute_input":"2023-10-15T20:02:36.202480Z","iopub.status.idle":"2023-10-15T20:02:38.627333Z","shell.execute_reply.started":"2023-10-15T20:02:36.202445Z","shell.execute_reply":"2023-10-15T20:02:38.626197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_test[0:6]","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.599679Z","iopub.execute_input":"2023-10-15T15:06:48.600558Z","iopub.status.idle":"2023-10-15T15:06:48.613431Z","shell.execute_reply.started":"2023-10-15T15:06:48.600526Z","shell.execute_reply":"2023-10-15T15:06:48.612403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'1065003664fe9233f9'=='1065003664fe9233f9'","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.615485Z","iopub.execute_input":"2023-10-15T15:06:48.616277Z","iopub.status.idle":"2023-10-15T15:06:48.628907Z","shell.execute_reply.started":"2023-10-15T15:06:48.616236Z","shell.execute_reply":"2023-10-15T15:06:48.627583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load data","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers import *  \nfrom keras.models import Model  ","metadata":{"execution":{"iopub.status.busy":"2023-10-15T16:25:12.256180Z","iopub.execute_input":"2023-10-15T16:25:12.256568Z","iopub.status.idle":"2023-10-15T16:25:12.263051Z","shell.execute_reply.started":"2023-10-15T16:25:12.256528Z","shell.execute_reply":"2023-10-15T16:25:12.261832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(path_list):\n    np_list = []\n    error_list=[]\n    for path in path_list:\n        elem=np.load(path, allow_pickle=True)\n        if elem.shape==(3, 121):\n            np_list.append(elem)\n        else:\n            print(path)\n            print(elem.shape)\n            error_list.append(path)\n    labels_files=[sub.replace('.npy', '.txt') for sub in error_list]\n    return np_list , labels_files","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.630762Z","iopub.execute_input":"2023-10-15T15:06:48.631397Z","iopub.status.idle":"2023-10-15T15:06:48.644855Z","shell.execute_reply.started":"2023-10-15T15:06:48.631363Z","shell.execute_reply":"2023-10-15T15:06:48.643709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_and_labels(path_list,text_path):\n    np_list = []\n    error_list=[]\n    labels = []\n    for path in path_list:\n        elem=np.load(path, allow_pickle=True)\n        if elem.shape==(3, 121):\n            np_list.append(elem)\n            text_file = open(path, 'r', encoding=\"utf8\", errors='ignore')\n            contents = text_file.read()\n            print (contents)\n#             contents = f.read()\n            labels.append(text_file.read()) \n            text_file.close()\n        else:\n            print(path)\n            print(elem.shape)\n            error_list.append(path)\n            text_file = open(path, 'r')\n            print(\"eticheta este:\",text_file.read())\n            text_file.close()\n    \n    return np_list,labels","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.646211Z","iopub.execute_input":"2023-10-15T15:06:48.646891Z","iopub.status.idle":"2023-10-15T15:06:48.661699Z","shell.execute_reply.started":"2023-10-15T15:06:48.646793Z","shell.execute_reply":"2023-10-15T15:06:48.660549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#for multiclass \ndef get_labels(file_list, labels_files):\n    labels = [] #-->my list variable\n    for file_path in file_list:\n        if file_path not in labels_files:\n            text_file = open(file_path, 'r')\n            labels.append(text_file.read()) \n            text_file.close()\n    labels = [int(i) for i in labels]\n    return labels","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:06:48.666197Z","iopub.execute_input":"2023-10-15T15:06:48.666952Z","iopub.status.idle":"2023-10-15T15:06:48.677462Z","shell.execute_reply.started":"2023-10-15T15:06:48.666907Z","shell.execute_reply":"2023-10-15T15:06:48.676530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Numpy dataset ready","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelBinarizer\nlabel_as_binary = LabelBinarizer()\n","metadata":{"execution":{"iopub.status.busy":"2023-10-15T17:40:32.707136Z","iopub.execute_input":"2023-10-15T17:40:32.707589Z","iopub.status.idle":"2023-10-15T17:40:32.792605Z","shell.execute_reply.started":"2023-10-15T17:40:32.707551Z","shell.execute_reply":"2023-10-15T17:40:32.791711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train,labels_files=load_data(train_files)\nX_train= np.asarray(X_train).astype('float32')\ny_train=get_labels(labels_train,labels_files)\n\ntrain_y_bin = label_as_binary.fit_transform(y_train)\nX_val,labels_files= load_data(validation_files)\nX_val=np.asarray(X_val).astype('float32')\ny_val=get_labels(labels_test,labels_files)\nval_y_bin= label_as_binary.fit_transform(y_val)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T17:42:01.407348Z","iopub.execute_input":"2023-10-15T17:42:01.407770Z","iopub.status.idle":"2023-10-15T17:42:46.352593Z","shell.execute_reply.started":"2023-10-15T17:42:01.407738Z","shell.execute_reply":"2023-10-15T17:42:46.351652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_train)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:08:27.969643Z","iopub.execute_input":"2023-10-15T15:08:27.969969Z","iopub.status.idle":"2023-10-15T15:08:27.976128Z","shell.execute_reply.started":"2023-10-15T15:08:27.969944Z","shell.execute_reply":"2023-10-15T15:08:27.975006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def unique(list1):\n    x = np.array(list1)\n    print(np.unique(x))\nunique(y_train)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:08:27.977565Z","iopub.execute_input":"2023-10-15T15:08:27.977896Z","iopub.status.idle":"2023-10-15T15:08:27.996909Z","shell.execute_reply.started":"2023-10-15T15:08:27.977869Z","shell.execute_reply":"2023-10-15T15:08:27.995571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.preprocessing import MultiLabelBinarizer\n# mlb = MultiLabelBinarizer()\n# mlb.fit(y_train)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:08:27.998307Z","iopub.execute_input":"2023-10-15T15:08:27.998802Z","iopub.status.idle":"2023-10-15T15:08:28.009998Z","shell.execute_reply.started":"2023-10-15T15:08:27.998774Z","shell.execute_reply":"2023-10-15T15:08:28.008662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:08:28.011370Z","iopub.execute_input":"2023-10-15T15:08:28.012116Z","iopub.status.idle":"2023-10-15T15:08:28.024456Z","shell.execute_reply.started":"2023-10-15T15:08:28.012072Z","shell.execute_reply":"2023-10-15T15:08:28.023674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=32\ntrain_dataset = tf.data.Dataset.from_tensor_slices((X_train, train_y_bin))\n#shuffle data https://www.tensorflow.org/api_docs/python/tf/data/Dataset#shuffle\ntrain_dataset = train_dataset.shuffle(train_dataset.cardinality())\ntrain_dataset = train_dataset.batch(batch_size)\ntest_dataset = tf.data.Dataset.from_tensor_slices((X_val, val_y_bin))\ntest_dataset=test_dataset.batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T17:45:40.118689Z","iopub.execute_input":"2023-10-15T17:45:40.119077Z","iopub.status.idle":"2023-10-15T17:45:40.147591Z","shell.execute_reply.started":"2023-10-15T17:45:40.119050Z","shell.execute_reply":"2023-10-15T17:45:40.146181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.cardinality().numpy()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T15:08:28.116372Z","iopub.execute_input":"2023-10-15T15:08:28.116693Z","iopub.status.idle":"2023-10-15T15:08:28.122942Z","shell.execute_reply.started":"2023-10-15T15:08:28.116665Z","shell.execute_reply":"2023-10-15T15:08:28.121988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"X_train shape:\", X_train.shape)\nprint(X_train.shape[0], \"train samples\")\nprint(X_val.shape[0], \"test samples\")","metadata":{"execution":{"iopub.status.busy":"2023-10-15T16:36:12.076155Z","iopub.execute_input":"2023-10-15T16:36:12.076532Z","iopub.status.idle":"2023-10-15T16:36:12.082561Z","shell.execute_reply.started":"2023-10-15T16:36:12.076504Z","shell.execute_reply":"2023-10-15T16:36:12.081212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape=(2,121)\nnum_classes=4","metadata":{"execution":{"iopub.status.busy":"2023-10-15T16:45:10.715192Z","iopub.execute_input":"2023-10-15T16:45:10.715561Z","iopub.status.idle":"2023-10-15T16:45:10.720302Z","shell.execute_reply.started":"2023-10-15T16:45:10.715533Z","shell.execute_reply":"2023-10-15T16:45:10.719368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.Sequential(\n    [\n        keras.Input(shape=input_shape),\n\n        layers.Dense(1024, activation=\"relu\"),\n        layers.Dropout(0.5),\n#         layers.Dense(512, activation=\"relu\"),\n#         layers.Dropout(0.5),\n        layers.Dense(256, activation=\"relu\"),\n        layers.Dropout(0.5),\n#         layers.Dense(128, activation=\"relu\"),\n#         layers.Dropout(0.5),\n        layers.Flatten(),\n        layers.Dense(num_classes, activation=\"softmax\"),\n    ]\n)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:20:57.991644Z","iopub.execute_input":"2023-10-15T19:20:57.992055Z","iopub.status.idle":"2023-10-15T19:20:58.109217Z","shell.execute_reply.started":"2023-10-15T19:20:57.992026Z","shell.execute_reply":"2023-10-15T19:20:58.108047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 128\nepochs = 75\n\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n\nmodel.fit(train_dataset, batch_size=batch_size, epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:21:04.842965Z","iopub.execute_input":"2023-10-15T19:21:04.843394Z","iopub.status.idle":"2023-10-15T19:27:16.863522Z","shell.execute_reply.started":"2023-10-15T19:21:04.843360Z","shell.execute_reply":"2023-10-15T19:27:16.862509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model2 = keras.Sequential(\n#     [\n#         keras.Input(shape=input_shape),\n#         layers.Conv1D(24,2, activation=\"relu\",data_format=\"channels_first\"),\n#         layers.MaxPooling1D(pool_size=2, strides=None, padding=\"valid\"),\n# #         layers.Conv1D(24,2, activation=\"relu\"),\n# #         layers.MaxPooling1D(pool_size=2, strides=None, padding=\"valid\"),\n#         layers.Flatten(),\n#         layers.Dropout(0.5),\n#         layers.Dense(num_classes, activation=\"softmax\"),\n#     ]\n# )\n\n# model2.summary()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T18:35:21.308848Z","iopub.execute_input":"2023-10-15T18:35:21.309345Z","iopub.status.idle":"2023-10-15T18:35:21.379957Z","shell.execute_reply.started":"2023-10-15T18:35:21.309303Z","shell.execute_reply":"2023-10-15T18:35:21.378916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Real data to predict","metadata":{}},{"cell_type":"code","source":"HOME_DIR = '/kaggle/working/' \nDATASET_PATH = 'dataset/signals120'\n!mkdir {HOME_DIR}dataset\n!mkdir {HOME_DIR}{DATASET_PATH}\n!mkdir {HOME_DIR}{DATASET_PATH}/real\npath_to_real=\"/kaggle/working/dataset/signals120/real/\"","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:07:10.465466Z","iopub.execute_input":"2023-10-15T19:07:10.465913Z","iopub.status.idle":"2023-10-15T19:07:13.962977Z","shell.execute_reply.started":"2023-10-15T19:07:10.465879Z","shell.execute_reply":"2023-10-15T19:07:13.961285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_real_test(real_test):\n    lenght=len(real_test)\n    maxrange=lenght//121\n    print(maxrange)\n    for window in np.arange(0, maxrange, 121, dtype=int):\n        temp=real_test.iloc[window:window+121]\n        scaled_data=scaler_model.transform(temp[['anglez','enmo']])\n        temp['anglez']= scaled_data[:,0]\n        temp['enmo']=scaled_data[:,1]\n        temp['convolve']=signal.fftconvolve(temp[\"anglez\"].values,temp[\"enmo\"].values,  mode='same')\n        serial=temp['series_id'].unique()[0]\n        enmo=temp['enmo'].to_numpy()\n        anglez=temp['anglez'].to_numpy()\n        convolve=temp['convolve'].to_numpy()\n        row=np.array([anglez, enmo,convolve], dtype='object')\n        filename=str(str(window)+serial)\n    #   print(\"fisier\",filename,\"nume\")\n        np.save(f'/kaggle/working/dataset/signals120/real/{filename}', row)\n        gc.collect\n        print(temp.shape)\n#         print(temp)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:18.961633Z","iopub.execute_input":"2023-10-15T19:12:18.962044Z","iopub.status.idle":"2023-10-15T19:12:18.969407Z","shell.execute_reply.started":"2023-10-15T19:12:18.962014Z","shell.execute_reply":"2023-10-15T19:12:18.968225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_real_series(path_to_real):\n    real_series = pd.read_parquet(path_to_real)\n    return real_series\nreal_test=get_real_series(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:19.934823Z","iopub.execute_input":"2023-10-15T19:12:19.935698Z","iopub.status.idle":"2023-10-15T19:12:19.949739Z","shell.execute_reply.started":"2023-10-15T19:12:19.935665Z","shell.execute_reply":"2023-10-15T19:12:19.948609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_test_series=real_test['series_id'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:20.383763Z","iopub.execute_input":"2023-10-15T19:12:20.384298Z","iopub.status.idle":"2023-10-15T19:12:20.390933Z","shell.execute_reply.started":"2023-10-15T19:12:20.384255Z","shell.execute_reply":"2023-10-15T19:12:20.389588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for series in real_test_series:\n    real_test=pd.read_parquet(\"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\", filters=[('series_id','=',series)])\n    make_real_test(real_test)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:20.871950Z","iopub.execute_input":"2023-10-15T19:12:20.872871Z","iopub.status.idle":"2023-10-15T19:12:20.895553Z","shell.execute_reply.started":"2023-10-15T19:12:20.872835Z","shell.execute_reply":"2023-10-15T19:12:20.894724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_numpy_labels_from_dir(datadir):\n    all_files = os.listdir(os.path.abspath(datadir))\n    data_files = list(filter(lambda file: file.endswith('.npy'), all_files))\n    label_files = list(filter(lambda file: file.endswith('.txt'), all_files))\n    data_files_path= [datadir+\"/\"+ s for s in data_files]\n    label_files_path= [datadir+\"/\"+ s for s in label_files]\n    data_files_path.sort()\n    label_files_path.sort()\n    return data_files_path , label_files_path","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:21.310985Z","iopub.execute_input":"2023-10-15T19:12:21.311798Z","iopub.status.idle":"2023-10-15T19:12:21.316866Z","shell.execute_reply.started":"2023-10-15T19:12:21.311767Z","shell.execute_reply":"2023-10-15T19:12:21.315833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_real=\"/kaggle/working/dataset/signals120/real/\"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_files,labels_real=get_numpy_labels_from_dir(path_to_real)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:12:21.903114Z","iopub.execute_input":"2023-10-15T19:12:21.903462Z","iopub.status.idle":"2023-10-15T19:12:21.909735Z","shell.execute_reply.started":"2023-10-15T19:12:21.903434Z","shell.execute_reply":"2023-10-15T19:12:21.907796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"real_files","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:16:29.308607Z","iopub.execute_input":"2023-10-15T19:16:29.309016Z","iopub.status.idle":"2023-10-15T19:16:29.316533Z","shell.execute_reply.started":"2023-10-15T19:16:29.308988Z","shell.execute_reply":"2023-10-15T19:16:29.315409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(path_list):\n    np_list = []\n    error_list=[]\n    for path in path_list:\n        elem=np.load(path, allow_pickle=True)\n        if elem.shape==(2, 121):\n            np_list.append(elem)\n        else:\n            print(path)\n            print(elem.shape)\n            error_list.append(path)\n    labels_files=[sub.replace('.npy', '.txt') for sub in error_list]\n    return np_list , labels_files","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:17:36.025532Z","iopub.execute_input":"2023-10-15T19:17:36.025916Z","iopub.status.idle":"2023-10-15T19:17:36.032654Z","shell.execute_reply.started":"2023-10-15T19:17:36.025885Z","shell.execute_reply":"2023-10-15T19:17:36.031529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_real,labels_files=load_data(real_files)\nX_real= np.asarray(X_real).astype('float32')","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:18:24.618639Z","iopub.execute_input":"2023-10-15T19:18:24.619033Z","iopub.status.idle":"2023-10-15T19:18:24.626384Z","shell.execute_reply.started":"2023-10-15T19:18:24.619003Z","shell.execute_reply":"2023-10-15T19:18:24.625036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(X_real)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:18:25.265603Z","iopub.execute_input":"2023-10-15T19:18:25.265957Z","iopub.status.idle":"2023-10-15T19:18:25.272412Z","shell.execute_reply.started":"2023-10-15T19:18:25.265931Z","shell.execute_reply":"2023-10-15T19:18:25.271595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=1\nreal_dataset = tf.data.Dataset.from_tensor_slices(X_real)\nreal_dataset=real_dataset.batch(batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:18:30.402374Z","iopub.execute_input":"2023-10-15T19:18:30.402917Z","iopub.status.idle":"2023-10-15T19:18:30.410384Z","shell.execute_reply.started":"2023-10-15T19:18:30.402889Z","shell.execute_reply":"2023-10-15T19:18:30.409569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = model.predict(real_dataset)\nprint(prediction)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:27:35.310419Z","iopub.execute_input":"2023-10-15T19:27:35.310826Z","iopub.status.idle":"2023-10-15T19:27:35.426202Z","shell.execute_reply.started":"2023-10-15T19:27:35.310793Z","shell.execute_reply":"2023-10-15T19:27:35.425028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score = model.evaluate(test_dataset, verbose=0)\nprint(\"Test loss:\", score[0])\nprint(\"Test accuracy:\", score[1])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:28:11.037823Z","iopub.execute_input":"2023-10-15T19:28:11.038204Z","iopub.status.idle":"2023-10-15T19:28:11.947220Z","shell.execute_reply.started":"2023-10-15T19:28:11.038174Z","shell.execute_reply":"2023-10-15T19:28:11.946102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import multilabel_confusion_matrix,ConfusionMatrixDisplay","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:41:42.782633Z","iopub.execute_input":"2023-10-15T19:41:42.782983Z","iopub.status.idle":"2023-10-15T19:41:42.787831Z","shell.execute_reply.started":"2023-10-15T19:41:42.782957Z","shell.execute_reply":"2023-10-15T19:41:42.786903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_values = np.concatenate([x for x, y in test_dataset], axis=0)\n\ntest_label = np.concatenate([y for x, y in test_dataset], axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:34:05.509997Z","iopub.execute_input":"2023-10-15T19:34:05.510384Z","iopub.status.idle":"2023-10-15T19:34:05.664944Z","shell.execute_reply.started":"2023-10-15T19:34:05.510353Z","shell.execute_reply":"2023-10-15T19:34:05.664071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_values)\nconfusion_matrix = multilabel_confusion_matrix(test_label, np.rint(y_pred))","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:43:02.563701Z","iopub.execute_input":"2023-10-15T19:43:02.564264Z","iopub.status.idle":"2023-10-15T19:43:03.304803Z","shell.execute_reply.started":"2023-10-15T19:43:02.564235Z","shell.execute_reply":"2023-10-15T19:43:03.303888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2023-10-15T19:43:05.621901Z","iopub.execute_input":"2023-10-15T19:43:05.622759Z","iopub.status.idle":"2023-10-15T19:43:05.630826Z","shell.execute_reply.started":"2023-10-15T19:43:05.622719Z","shell.execute_reply":"2023-10-15T19:43:05.629581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}