{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Aida Ebadi and Félicie Levinton","metadata":{"id":"igVBgnjr5v7y"}},{"cell_type":"markdown","source":"\n\nSleep Spindles and K-complexes, where\nboth exclusively occur in the second sleep stage.\n\nW: Awake state (stage W) is characterized by alpha or faster\nfrequency bands occupying more than 50% of the epoch,\nfrequent eye movements and high EMG tone.\n• N1: Stage N1 is scored when alpha occupies more than 50%\nof epoch while theta activity, slow rolling eye movements and\nvertex waves are evident.\n• N2: Stage N2 is scored when sleep spindles or K-complexes\n(less than 3 min apart) are noted.\n• N3: Stage N3 is characterized by delta activity detected in over\n20% of the epoch length.\n• REM: Upon sleep scoring an epoch is marked as REM when\nsaw-tooth waves along with rapid eye movements as well as\nlowest EMG signals are observed through each epoch.\n\n","metadata":{"id":"bio7OAKlxuvy"}},{"cell_type":"code","source":"cd \"/kaggle/input/dreem-automated-sleep-staging/\"","metadata":{"id":"uJDig5E8X4UL","outputId":"628aba27-3d9d-4fa3-81e1-2988b17af8b1","execution":{"iopub.status.busy":"2023-01-09T09:43:53.646388Z","iopub.execute_input":"2023-01-09T09:43:53.646801Z","iopub.status.idle":"2023-01-09T09:43:53.652013Z","shell.execute_reply.started":"2023-01-09T09:43:53.646769Z","shell.execute_reply":"2023-01-09T09:43:53.651096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom datetime import datetime\nfrom sklearn.metrics import balanced_accuracy_score, cohen_kappa_score, confusion_matrix\nimport matplotlib.pyplot as plt\nimport json\n\nimport os\nimport numpy as np\nimport h5py\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom datetime import datetime\nfrom scipy.signal import spectrogram\nfrom sklearn.metrics import balanced_accuracy_score, cohen_kappa_score, confusion_matrix\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport random as rd\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\n\nfrom scipy.stats import skew\nfrom scipy.stats import kurtosis\n","metadata":{"id":"9sjXBak0ZWg9","execution":{"iopub.status.busy":"2023-01-09T09:43:53.659814Z","iopub.execute_input":"2023-01-09T09:43:53.660221Z","iopub.status.idle":"2023-01-09T09:43:53.671699Z","shell.execute_reply.started":"2023-01-09T09:43:53.660183Z","shell.execute_reply":"2023-01-09T09:43:53.670405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"EEG is sampled at 250 Hz \n\n3 acc sampled at 50 Hz\n\nWake: Alpha\nN3: Slow waves\nN2: Spindle\nREM: eye movment \n**\n\nEach NumPy array is a table with N row and 42001 columns.\n\nColumn 0: index of the epoch in the record, the i-th epoch corresponds to the data from t=i * 30 seconds to t=(i+1) seconds.\n","metadata":{"id":"De2M5JqhaNlj"}},{"cell_type":"code","source":"def min_max_d(l):\n  import math\n  r = []\n  for k in l:\n    a2 = math.pow(max(k) - min(k), 2)\n    b2 = math.pow(np.argmax(k) - np.argmin(k), 2)\n    r.append(math.sqrt( a2 + b2 ))\n  return r\n\nfrequency_bands = {\n        \"delta\": [0.5, 4],\n        \"theta\": [4, 8],\n        \"alpha\": [8, 12],\n       \"sigma\": [12, 16],\n       \"beta\": [16, 30],\n    }\n\nstatistics = {\n    \"stdev\":lambda x:np.std(x,1),\n    \"mean\":lambda x:np.mean(np.abs(x),1),\n    \"minmax\":lambda x:min_max_d(x)\n  }\n\naccelerometer = {\n    \"stdev_acc\":lambda x:np.std(x,1),\n    \"mean_acc\":lambda x:np.mean(np.abs(x),1),\n    \"var_acc\":lambda x:np.var(x,1),\n    \"minmax\":lambda x:min_max_d(x)\n} \n\nvariable_lists = list(frequency_bands) + list(statistics) + list(accelerometer)\n\ndef get_relative_spectral_power_for_epoch(data):\n    \"\"\"\n    Compute the relative spectral power for each of the frequency bands defined above\n    \"\"\"\n    EEG_FS = 250\n    psds = []\n    sfreqs,t,psd = spectrogram(data,250,nperseg = 1000,noverlap = 750)\n    psd = np.mean(np.abs(psd),-1)\n    spectral_power_band = {}\n    for name, freqband in frequency_bands.items():\n        spec_power = psd[:,(sfreqs >= freqband[0]) & (sfreqs < freqband[1])]\n        spec_power = np.sum(spec_power, 1)\n        spectral_power_band[name] = spec_power / np.sum(psd,1)\n    return spectral_power_band\n\ndef compute_stats(data):\n    \"\"\"\n    Compute the statistics of a signal\n    \"\"\"\n    EEG_FS = 250\n    result = {k:f(data) for k,f in statistics.items()}\n    return result","metadata":{"id":"uy4t7BtbRPBo","execution":{"iopub.status.busy":"2023-01-09T09:43:53.673228Z","iopub.execute_input":"2023-01-09T09:43:53.674191Z","iopub.status.idle":"2023-01-09T09:43:53.691202Z","shell.execute_reply.started":"2023-01-09T09:43:53.674152Z","shell.execute_reply":"2023-01-09T09:43:53.690132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_stats_acc(data):\n    \"\"\"\n    Compute the statistics of a signal\n    \"\"\"\n    EEG_FS = 250\n    result = {k:f(data) for k,f in accelerometer.items()}\n    return result","metadata":{"id":"fJbf7p9BOfT9","execution":{"iopub.status.busy":"2023-01-09T09:43:53.692760Z","iopub.execute_input":"2023-01-09T09:43:53.693548Z","iopub.status.idle":"2023-01-09T09:43:53.712411Z","shell.execute_reply.started":"2023-01-09T09:43:53.693512Z","shell.execute_reply":"2023-01-09T09:43:53.711090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Column 37501:39001: X-axis accelerometer channel sampled at 50Hz\nColumn 39001:40501: Y-axis accelerometer channel sampled at 50Hz\nColumn 40501:42001: Z-axis accelerometer channel sampled at 50Hz","metadata":{"id":"FOSxSgo8fT7z"}},{"cell_type":"code","source":"data_for_records = {}\nhypnogram_for_records = {}\nhypnograms = pd.read_csv('targets_train.csv')\ni=0\nfor record in os.listdir(\"training_records\"):\n    print(record)\n    record_number = int(record[-5])\n    x = np.load(f'training_records/{record}')\n    data_for_record = x[:,1:250 * 30 + 1]\n    data_accelero=x[:,39001:42001]\n    data_for_records[record] = get_relative_spectral_power_for_epoch(data_for_record)\n    data_for_records[record].update(compute_stats(data_for_record))\n    data_for_records[record].update(compute_stats_acc(data_accelero))\n    hypnogram_for_records[record] = list(hypnograms[hypnograms['record'] == record_number]['target'])\n\nrecords_list = list(data_for_records)\ntraining_record = records_list[:]\n\ndata_for_records_test = {}\nfor record in os.listdir(\"test_records\"):\n    print(record)\n    record_number = int(record[-5])\n    x = np.load(f'test_records/{record}')\n    data_for_record = x[:,1:250 * 30 + 1]\n    data_accelero=x[:,39001:42001]\n    data_for_records_test[record] = get_relative_spectral_power_for_epoch(data_for_record)\n    data_for_records_test[record].update(compute_stats(data_for_record))\n    data_for_records_test[record].update(compute_stats_acc(data_accelero))\n    hypnogram_for_records[record] = list(hypnograms[hypnograms['record'] == record_number]['target'])\n\nrecords_list = list(data_for_records_test)\ndata_for_records.update(data_for_records_test)\ntest_records = records_list[:]\n\nprint('Training records: ',training_record)\nprint('Test records: ', test_records)\nprint(data_for_records.keys())\n\n#variable_list = list(data_for_records[record].keys())","metadata":{"id":"r4q9Th4cRSCA","execution":{"iopub.status.busy":"2023-01-09T09:43:53.714017Z","iopub.execute_input":"2023-01-09T09:43:53.714612Z","iopub.status.idle":"2023-01-09T09:44:14.620831Z","shell.execute_reply.started":"2023-01-09T09:43:53.714575Z","shell.execute_reply":"2023-01-09T09:44:14.619526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test avec un filtre de butterworth","metadata":{"id":"FV33IBxZW2bC"}},{"cell_type":"code","source":"from scipy import signal\ndef data(c1,c2):\n  data_for_records = {}\n  hypnogram_for_records = {}\n  hypnograms = pd.read_csv('targets_train.csv')\n  i=0\n  l = [(\"test_records\",x) for x in os.listdir(\"test_records\")]\n  l += [(\"training_records\",x) for x in os.listdir(\"training_records\")]\n  for dir,record in l:\n      record_number = int(record[-5])\n      x = np.load(f'{dir}/{record}')\n      sig = x[:,c1:c2]\n      sos = signal.butter(4, (0.5,50), btype='band', fs=1000, output='sos')\n      data_for_record = signal.sosfilt(sos, sig)\n      #data_accelero=x[:,39001:42001]\n      data_for_records[record] = get_relative_spectral_power_for_epoch(data_for_record)\n      data_for_records[record].update(compute_stats(data_for_record))\n      #data_for_records[record].update(compute_stats_acc(data_accelero))\n      hypnogram_for_records[record] = list(hypnograms[hypnograms['record'] == record_number]['target'])\n\n  return(data_for_records,hypnogram_for_records)","metadata":{"execution":{"iopub.status.busy":"2023-01-09T09:49:14.428634Z","iopub.execute_input":"2023-01-09T09:49:14.429060Z","iopub.status.idle":"2023-01-09T09:49:14.439349Z","shell.execute_reply.started":"2023-01-09T09:49:14.429025Z","shell.execute_reply":"2023-01-09T09:49:14.438214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy import signal\ndef data_non_filtered(c1,c2):\n  data_for_records = {}\n  hypnogram_for_records = {}\n  hypnograms = pd.read_csv('targets_train.csv')\n  i=0\n  l = [(\"test_records\",x) for x in os.listdir(\"test_records\")]\n  l += [(\"training_records\",x) for x in os.listdir(\"training_records\")]\n  for dir,record in l:\n      record_number = int(record[-5])\n      x = np.load(f'{dir}/{record}')\n      data_for_record = x[:,c1:c2]\n      #data_accelero=x[:,39001:42001]\n      data_for_records[record] = get_relative_spectral_power_for_epoch(data_for_record)\n      data_for_records[record].update(compute_stats(data_for_record))\n      #data_for_records[record].update(compute_stats_acc(data_accelero))\n      hypnogram_for_records[record] = list(hypnograms[hypnograms['record'] == record_number]['target'])\n\n  return(data_for_records,hypnogram_for_records)","metadata":{"id":"Nq4M0pqLWkDQ","execution":{"iopub.status.busy":"2023-01-09T09:44:14.635473Z","iopub.execute_input":"2023-01-09T09:44:14.636006Z","iopub.status.idle":"2023-01-09T09:44:14.653574Z","shell.execute_reply.started":"2023-01-09T09:44:14.635969Z","shell.execute_reply":"2023-01-09T09:44:14.652602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for accelerometer","metadata":{"id":"42e8Tt_-EQs1"}},{"cell_type":"code","source":"def data_acc(c1,c2):\n  data_for_records = {}\n  hypnogram_for_records = {}\n  hypnograms = pd.read_csv('targets_train.csv')\n  i=0\n  l = [(\"test_records\",x) for x in os.listdir(\"test_records\")]\n  l += [(\"training_records\",x) for x in os.listdir(\"training_records\")]\n  for dir,record in l:\n      record_number = int(record[-5])\n      x = np.load(f'{dir}/{record}')\n      #x = np.load(f'training_records/{record}')\n      data_for_record = x[:,c1:c2]\n      data_for_records[record] = compute_stats_acc(data_for_record)\n      hypnogram_for_records[record] = list(hypnograms[hypnograms['record'] == record_number]['target'])\n\n  return(data_for_records,hypnogram_for_records)","metadata":{"id":"gECjiL4zxBCy","execution":{"iopub.status.busy":"2023-01-09T09:44:14.656251Z","iopub.execute_input":"2023-01-09T09:44:14.656673Z","iopub.status.idle":"2023-01-09T09:44:14.671676Z","shell.execute_reply.started":"2023-01-09T09:44:14.656615Z","shell.execute_reply":"2023-01-09T09:44:14.670328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#eeg=data(1,37501)[0]\neeg=[]\neeg.append(data(1,7500)[0]) #ligne 1 = EEG1\n#eeg1=data(1,7500)[0]\neeg.append(data(7501,15000)[0])\neeg.append(data(15001,22500)[0])\neeg.append(data(22501,30000)[0])\neeg.append(data(30001,37501)[0])\n","metadata":{"id":"ByzTkFSEivdJ","execution":{"iopub.status.busy":"2023-01-09T09:49:20.939518Z","iopub.execute_input":"2023-01-09T09:49:20.939962Z","iopub.status.idle":"2023-01-09T09:50:46.281142Z","shell.execute_reply.started":"2023-01-09T09:49:20.939930Z","shell.execute_reply":"2023-01-09T09:50:46.279765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyp=data(1,7500)[1]","metadata":{"id":"IBHnuOF48LZX","execution":{"iopub.status.busy":"2023-01-09T09:51:21.728277Z","iopub.execute_input":"2023-01-09T09:51:21.728694Z","iopub.status.idle":"2023-01-09T09:51:38.857704Z","shell.execute_reply.started":"2023-01-09T09:51:21.728658Z","shell.execute_reply":"2023-01-09T09:51:38.856429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc_xyz=data_acc(37501,42001)[0]\nhyp_xyz=data_acc(37501,42001)[1]","metadata":{"id":"EHrKRrHAs-Us","execution":{"iopub.status.busy":"2023-01-09T09:45:29.650034Z","iopub.execute_input":"2023-01-09T09:45:29.651119Z","iopub.status.idle":"2023-01-09T09:45:43.783975Z","shell.execute_reply.started":"2023-01-09T09:45:29.651078Z","shell.execute_reply":"2023-01-09T09:45:43.782584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(acc_xyz.keys())\nprint(hyp.keys())\nprint(hyp_xyz.keys())","metadata":{"execution":{"iopub.status.busy":"2023-01-09T09:45:43.785573Z","iopub.execute_input":"2023-01-09T09:45:43.786552Z","iopub.status.idle":"2023-01-09T09:45:43.793071Z","shell.execute_reply.started":"2023-01-09T09:45:43.786498Z","shell.execute_reply":"2023-01-09T09:45:43.791680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#rd.seed(1234)\n#records_list = list(data_for_records)\n#rd.shuffle(records_list)\n#training_record,test_records = records_list[:4],records_list[4:]\n#print('Training records: ',training_record)\n#print('Test records: ', test_records)","metadata":{"id":"7NpshKYuRViM","outputId":"2fce585e-ada3-4a2c-8ebb-9591bc7c4dbe","execution":{"iopub.status.busy":"2023-01-09T09:45:43.794945Z","iopub.execute_input":"2023-01-09T09:45:43.795494Z","iopub.status.idle":"2023-01-09T09:45:43.809382Z","shell.execute_reply.started":"2023-01-09T09:45:43.795444Z","shell.execute_reply":"2023-01-09T09:45:43.806955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_dataset(records, data_for_records,hypnogram_for_records):\n    X,y = [],[]\n    for record in records:\n        X_for_record = np.expand_dims(np.array([value for value in data_for_records[record].values()]).T,0)\n        y.extend(hypnogram_for_records[record])\n        X.extend(X_for_record)\n    return np.concatenate(X),y\n\n\n#X_train,y_train = build_dataset(training_record,data_for_records,hypnogram_for_records)\n#X_test,y_test = build_dataset(test_records,data_for_records,hypnogram_for_records)\n\n","metadata":{"id":"PjpFItaCSIe6","execution":{"iopub.status.busy":"2023-01-09T09:52:01.491756Z","iopub.execute_input":"2023-01-09T09:52:01.492133Z","iopub.status.idle":"2023-01-09T09:52:01.499206Z","shell.execute_reply.started":"2023-01-09T09:52:01.492103Z","shell.execute_reply":"2023-01-09T09:52:01.498003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print([len(v) for k,v in hypnogram_for_records.items()])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T09:45:43.827991Z","iopub.execute_input":"2023-01-09T09:45:43.828387Z","iopub.status.idle":"2023-01-09T09:45:43.842316Z","shell.execute_reply.started":"2023-01-09T09:45:43.828354Z","shell.execute_reply":"2023-01-09T09:45:43.840593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We applly a random forest on each signal (eeg1, eeg2, ... eeg 5 and accelerometry x, y and z)\n\nideas for after : as it is known that there are proportionally fewer L1 sleep states than the other states during a normal night there is a bias in the training dataset. We will try to improve class imbalance using Class Weights random forest,  K-Fold Cross Validation or Stratified K-Fold Cross Validation?","metadata":{"id":"xWOgmDimHsSH"}},{"cell_type":"code","source":"Xy_train = []\nXy_test = []\nfor i in range(5):\n  Xy_train.append(build_dataset(training_record,eeg[i],hyp))\n  Xy_test.append(build_dataset(test_records,eeg[i],hyp))\nXy_train.append(build_dataset(training_record,acc_xyz,hyp_xyz))\nXy_test.append(build_dataset(test_records,acc_xyz,hyp_xyz))\n\n\n\n#On importe l'algorithme à partir de sklearn\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import RandomForestClassifier\n#On créé un Random Forest de 100 arbres \nrf=[]\nfor i in range(6):\n  #try with weights to improve N1 detection\n  rf.append(RandomForestClassifier(n_estimators=1000 ,random_state=0))#, class_weight = {0:0.5, 1: 5, 2: 0.5, 3: 0.2, 4: 0.1}))\n  rf[i].fit(Xy_train[i][0],Xy_train[i][1])\n","metadata":{"id":"Z_0aHfLmHrs-","execution":{"iopub.status.busy":"2023-01-09T10:07:08.836991Z","iopub.execute_input":"2023-01-09T10:07:08.837402Z","iopub.status.idle":"2023-01-09T10:08:37.054570Z","shell.execute_reply.started":"2023-01-09T10:07:08.837370Z","shell.execute_reply":"2023-01-09T10:08:37.053073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(rf)\npredictions=[]\nfor i in range (6):\n  predictions.append(rf[i].predict(Xy_test[i][0]))\n  #predictions.append(clf[i].predict(Xy_test[i][0]))\nprint ([len(x) for x in predictions])","metadata":{"id":"lBv9J2j8_d6M","outputId":"8c0f3371-71fe-4a38-a5be-d5dbc93a27a2","execution":{"iopub.status.busy":"2023-01-09T10:09:46.300159Z","iopub.execute_input":"2023-01-09T10:09:46.300596Z","iopub.status.idle":"2023-01-09T10:09:51.177188Z","shell.execute_reply.started":"2023-01-09T10:09:46.300550Z","shell.execute_reply":"2023-01-09T10:09:51.175847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.metrics import plot_confusion_matrix\n\n#for i in range(6):\n  #plot_confusion_matrix(rf[i], Xy_test[i][0],Xy_test[i][1],display_labels = ['Wake','N1','N2','N3','REM'])","metadata":{"id":"Zz94wc0iJzfy","outputId":"29977ce9-96d1-4080-af69-1e97135ae269","execution":{"iopub.status.busy":"2023-01-09T09:45:43.864186Z","iopub.status.idle":"2023-01-09T09:45:43.866158Z","shell.execute_reply.started":"2023-01-09T09:45:43.865604Z","shell.execute_reply":"2023-01-09T09:45:43.865675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"for each epoch, we take as prediction the sleep state on which the most predicted by the random forest : we take the maximum occurrence ","metadata":{"id":"VvTLrx1uxtL1"}},{"cell_type":"code","source":"from sklearn.utils import all_estimators\nnew_predictions = []\nfor i in range(len(predictions[0])):\n  all_predict = [prediction[i] for prediction in predictions]\n  new_predictions.append(1 if sum(1 if x==1 else 0 for x in all_predict)>=2 else max(all_predict, key=all_predict.count))\nnew_predictions = np.array(new_predictions)\nprint(new_predictions)\nprint(len(new_predictions))","metadata":{"id":"I7CKqBa5nPPA","outputId":"ee4591f3-941a-44b6-9a2f-187da8ce8857","execution":{"iopub.status.busy":"2023-01-09T10:09:54.395470Z","iopub.execute_input":"2023-01-09T10:09:54.396445Z","iopub.status.idle":"2023-01-09T10:09:54.420096Z","shell.execute_reply.started":"2023-01-09T10:09:54.396401Z","shell.execute_reply":"2023-01-09T10:09:54.418735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We compare the prediction of the model defined with the hypnogram: ","metadata":{"id":"Ino3WOBh4C2_"}},{"cell_type":"code","source":"#comp=new_predictions-Xy_test[0][1]","metadata":{"id":"T4QkBAvLPl7P","execution":{"iopub.status.busy":"2023-01-09T09:45:43.872176Z","iopub.status.idle":"2023-01-09T09:45:43.873190Z","shell.execute_reply.started":"2023-01-09T09:45:43.872887Z","shell.execute_reply":"2023-01-09T09:45:43.872929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def somme(liste):\n    _somme = 0\n    for i in liste:\n        _somme = _somme + i\n    return _somme","metadata":{"id":"Ji-inl5CPrtW","execution":{"iopub.status.busy":"2023-01-09T09:45:43.874935Z","iopub.status.idle":"2023-01-09T09:45:43.876238Z","shell.execute_reply.started":"2023-01-09T09:45:43.875944Z","shell.execute_reply":"2023-01-09T09:45:43.875972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#numberOfZeros = sum(comp[:]==0)\n#accurate = numberOfZeros\n#total = len(comp)\n#print(accurate) #number of good prediction\n#print(total) #total numbeer of predictions\n#print(len(comp)-numberOfZeros)#nomber of bad predictions\n#precision=accurate/total*100\n\n#print('accuracy=',precision)","metadata":{"id":"OV4uyc3rPwSp","outputId":"3bd3c4ad-ada0-4101-a8cf-c0176e7f2de7","execution":{"iopub.status.busy":"2023-01-09T09:45:43.877718Z","iopub.status.idle":"2023-01-09T09:45:43.878481Z","shell.execute_reply.started":"2023-01-09T09:45:43.878237Z","shell.execute_reply":"2023-01-09T09:45:43.878267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Results**","metadata":{"id":"0s20q0pi4hbL"}},{"cell_type":"code","source":"#confusion matrix\n#from sklearn import metrics\n#confusion_matrix = metrics.confusion_matrix(Xy_test[0][1], new_predictions)\n#cm_display = metrics.ConfusionMatrixDisplay(confusion_matrix = confusion_matrix, display_labels = ['Wake','N1','N2','N3','REM'])\n#cm_display.plot()\n#plt.show()\n\n\n#from sklearn.metrics import cohen_kappa_score\n#scores\n#accuracy=precision\n#kappa=cohen_kappa_score(Xy_test[0][1], new_predictions)\n#print('accuracy=',precision,'%')\n#print('cohen kappa score=',kappa)\n","metadata":{"id":"-tNy-0DrTFta","outputId":"6e8d62b9-cace-49ac-b42d-72f59e8ac072","execution":{"iopub.status.busy":"2023-01-09T09:45:43.880390Z","iopub.status.idle":"2023-01-09T09:45:43.881122Z","shell.execute_reply.started":"2023-01-09T09:45:43.880887Z","shell.execute_reply":"2023-01-09T09:45:43.880911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print([len(data_for_records[i][\"delta\"]) for i in test_records])","metadata":{"execution":{"iopub.status.busy":"2023-01-09T09:45:43.882215Z","iopub.status.idle":"2023-01-09T09:45:43.884099Z","shell.execute_reply.started":"2023-01-09T09:45:43.883591Z","shell.execute_reply":"2023-01-09T09:45:43.883703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_records)\nprint(len(new_predictions))\nprint(sum([len(data_for_records[i][\"delta\"]) for i in test_records]))\ncombined_length = [0]\nfor i,record in enumerate(test_records):\n  combined_length.append(combined_length[-1] + len(data_for_records[record][\"delta\"]))\ncombined_length = combined_length[1:]\nprint(combined_length)","metadata":{"id":"Bm0b5Jc05GIe","outputId":"ad74bebb-86db-47bb-ec29-806f8dad8641","execution":{"iopub.status.busy":"2023-01-09T10:10:00.515720Z","iopub.execute_input":"2023-01-09T10:10:00.516216Z","iopub.status.idle":"2023-01-09T10:10:00.525658Z","shell.execute_reply.started":"2023-01-09T10:10:00.516174Z","shell.execute_reply":"2023-01-09T10:10:00.524268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = new_predictions\nindex = 0\npredictions = []\nrecord_number = int(test_records[index][7])\nj = 0\nfor i, pred in enumerate(preds):\n  if i >= combined_length[index]:\n    index += 1\n    record_number = int(test_records[index][7])\n    j = 0\n    print(\"Changing record number:\", record_number)\n  predictions.append({\"identifier\":record_number * 10000 + j,'target':pred})\n  j += 1\npredictions = pd.DataFrame(predictions)\nprint(predictions)","metadata":{"id":"ixhudMkv6hI6","outputId":"164c763d-4854-4976-9cea-d77fd7e9faca","execution":{"iopub.status.busy":"2023-01-09T10:10:03.612592Z","iopub.execute_input":"2023-01-09T10:10:03.613053Z","iopub.status.idle":"2023-01-09T10:10:03.634894Z","shell.execute_reply.started":"2023-01-09T10:10:03.613020Z","shell.execute_reply":"2023-01-09T10:10:03.633506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions.to_csv('/kaggle/working/submission.csv',index = None)","metadata":{"id":"eTI-L4Cb6lYE","outputId":"42f4ce6f-03a3-472c-bd72-9f506698a73d","execution":{"iopub.status.busy":"2023-01-09T10:10:06.714521Z","iopub.execute_input":"2023-01-09T10:10:06.715808Z","iopub.status.idle":"2023-01-09T10:10:06.726251Z","shell.execute_reply.started":"2023-01-09T10:10:06.715757Z","shell.execute_reply":"2023-01-09T10:10:06.724515Z"},"trusted":true},"execution_count":null,"outputs":[]}]}