{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importance of Resample the data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nfrom scipy import signal","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:41:29.357601Z","iopub.execute_input":"2022-09-16T20:41:29.358149Z","iopub.status.idle":"2022-09-16T20:41:29.938252Z","shell.execute_reply.started":"2022-09-16T20:41:29.358054Z","shell.execute_reply":"2022-09-16T20:41:29.937273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.load('../input/eeg-data-from-passive-viewing/clr_EEG32.npy')\nlabels = np.load('../input/eeg-data-from-passive-viewing/clr_Labels.npy')","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:05:32.982786Z","iopub.execute_input":"2022-09-16T04:05:32.983813Z","iopub.status.idle":"2022-09-16T04:05:42.636593Z","shell.execute_reply.started":"2022-09-16T04:05:32.983736Z","shell.execute_reply":"2022-09-16T04:05:42.635331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#chans x ms x epochs\nprint(data.shape)\nprint(labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:05:44.064559Z","iopub.execute_input":"2022-09-16T04:05:44.065356Z","iopub.status.idle":"2022-09-16T04:05:44.071264Z","shell.execute_reply.started":"2022-09-16T04:05:44.065311Z","shell.execute_reply":"2022-09-16T04:05:44.069953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data.transpose(1,0,2).shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:05:45.654924Z","iopub.execute_input":"2022-09-16T04:05:45.655318Z","iopub.status.idle":"2022-09-16T04:05:45.664831Z","shell.execute_reply.started":"2022-09-16T04:05:45.655289Z","shell.execute_reply":"2022-09-16T04:05:45.663136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 683\nprint(labels[idx])\nplt.plot(data.transpose(1,0,2)[500:800,0, idx])","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:05:47.080828Z","iopub.execute_input":"2022-09-16T04:05:47.081240Z","iopub.status.idle":"2022-09-16T04:05:47.342452Z","shell.execute_reply.started":"2022-09-16T04:05:47.081205Z","shell.execute_reply":"2022-09-16T04:05:47.340985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 683\nprint(labels[idx])\nplt.plot(data.transpose(1,0,2)[500:650,0, idx])","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:08:32.672332Z","iopub.execute_input":"2022-09-16T04:08:32.672736Z","iopub.status.idle":"2022-09-16T04:08:32.894203Z","shell.execute_reply.started":"2022-09-16T04:08:32.672704Z","shell.execute_reply":"2022-09-16T04:08:32.892991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 683\nprint(labels[idx])\nwave = signal.resample(data.transpose(1,0,2)[500:800,0, idx], 150)\nplt.plot(wave)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T04:13:07.426589Z","iopub.execute_input":"2022-09-16T04:13:07.427389Z","iopub.status.idle":"2022-09-16T04:13:07.650863Z","shell.execute_reply.started":"2022-09-16T04:13:07.427313Z","shell.execute_reply":"2022-09-16T04:13:07.649515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The color trails are labeled as [7,8,9,10,11,12,13] where 7 denotes a trial that was marked by experts as containing an artifact.\n\nThe orientation trials are labeled as [1,2,3,4,5,6,14] where 14 denotes a trial that was marked by experts as containing an artifact.\n","metadata":{}},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"%pip install pyod\nimport numpy as np\nfrom pyod.models import hbos\nfrom pyod.utils.data import evaluate_print\nfrom sklearn.metrics import confusion_matrix,cohen_kappa_score,f1_score\nfrom scipy import signal\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:41:32.917144Z","iopub.execute_input":"2022-09-16T20:41:32.917549Z","iopub.status.idle":"2022-09-16T20:41:49.059541Z","shell.execute_reply.started":"2022-09-16T20:41:32.917518Z","shell.execute_reply":"2022-09-16T20:41:49.057793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#####\n# Trial or epoch = eeg segment\n#####\n\nchannels = 32\nselected_channels = 32\ndataset_trial_duration = 1200 # Original dataset size of each trial (1.2 seconds)\nfs = 1000.0        # Original dataset eeg sampling rate\nseconds = 0.300    # Size of each segment we want\nnew_fs = 500.0     # Resize the dataset eeg sampling rate to 250.0 Hz\ninitial_time = 500 # We aren't going to pick the firsts 500ms because don't contain noise\nsave_model = False","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:08.469023Z","iopub.execute_input":"2022-09-16T20:42:08.469441Z","iopub.status.idle":"2022-09-16T20:42:08.475656Z","shell.execute_reply.started":"2022-09-16T20:42:08.469403Z","shell.execute_reply":"2022-09-16T20:42:08.474686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGDataset:\n    def __init__(self, data, datalabels, labels, artifact_label, seconds=0.5, fs=500.0, initial_time=200, channels=22):\n        self.data = np.load(data)\n        self.datalabels = np.load(datalabels)\n        \n        self.labels = labels\n        self.artifact_label = artifact_label\n        \n        self.seconds = seconds\n        self.fs = fs \n        self.duration = int(self.fs * self.seconds)\n        self.initial_time = initial_time\n        self.channels = channels\n        \n    def get_trials_from_channels(self):\n        trials = []\n        classes = []\n        artifacts = []\n        \n        for i in range(self.data.shape[2]):\n            class_ = self.datalabels[i]\n            if class_ not in self.labels:\n                continue\n            # [channels x duration, trials]\n            trial = self.data[:self.channels,self.initial_time:self.duration+self.initial_time,i]\n            trials.append(trial)\n            classes.append(class_)\n            \n            if class_ == self.artifact_label: artifacts.append(1)\n            else: artifacts.append(0)\n\n        trials = np.array(trials)\n        # feature_arr = extractFeatures(trials)\n                \n        return trials, classes, artifacts","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:10.215515Z","iopub.execute_input":"2022-09-16T20:42:10.216320Z","iopub.status.idle":"2022-09-16T20:42:10.227978Z","shell.execute_reply.started":"2022-09-16T20:42:10.216270Z","shell.execute_reply":"2022-09-16T20:42:10.226675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# We want trials of 0.5 seconds or 500ms\n# An trial array of EEG in 1000 Hz of 0.5s is size 500\n# We will resample the data to 500 Hz and we want 0.5s for each trial\n# So each trial array will have size of 250 (0.5s in 500 Hz)\n\n\nprint(\"Data contain 32 channels\")\nprint(\"Using data with duration of:\",seconds,\"seconds\")\nprint(\"Sampling Frequency:\",fs,\"Hz\")\nprint(\"Our trial Length (seconds * fs) =\",fs*seconds)\nprint(\"We will resample the data frequency to:\", new_fs)\nprint(\"So our trial length will be:\",new_fs*seconds)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:12.699278Z","iopub.execute_input":"2022-09-16T20:42:12.699658Z","iopub.status.idle":"2022-09-16T20:42:12.706913Z","shell.execute_reply.started":"2022-09-16T20:42:12.699628Z","shell.execute_reply":"2022-09-16T20:42:12.705270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TRAIN DATASET ##\ntrain_data_path = '../input/eeg-data-from-passive-viewing/clr_EEG32.npy'\ntrain_datalabels_path = '../input/eeg-data-from-passive-viewing/clr_Labels.npy'\ntrain_labels = [7,8,9,10,11,12,13]\ntrain_artifact_label = 7","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:14.385508Z","iopub.execute_input":"2022-09-16T20:42:14.386116Z","iopub.status.idle":"2022-09-16T20:42:14.391150Z","shell.execute_reply.started":"2022-09-16T20:42:14.386082Z","shell.execute_reply":"2022-09-16T20:42:14.390046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = EEGDataset(train_data_path, train_datalabels_path, train_labels, train_artifact_label, seconds, fs, initial_time, selected_channels)\ntrain_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:15.839218Z","iopub.execute_input":"2022-09-16T20:42:15.839903Z","iopub.status.idle":"2022-09-16T20:42:23.863513Z","shell.execute_reply.started":"2022-09-16T20:42:15.839866Z","shell.execute_reply":"2022-09-16T20:42:23.862309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TEST DATASET ##\ntest_data_path = '../input/eeg-data-from-passive-viewing/ori_EEG32.npy'\ntest_datalabels_path = '../input/eeg-data-from-passive-viewing/ori_Labels.npy'\ntest_labels = [1,2,3,4,5,6,14]\ntest_artifact_label = 14","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:23.865481Z","iopub.execute_input":"2022-09-16T20:42:23.866121Z","iopub.status.idle":"2022-09-16T20:42:23.872178Z","shell.execute_reply.started":"2022-09-16T20:42:23.866084Z","shell.execute_reply":"2022-09-16T20:42:23.870959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = EEGDataset(test_data_path, test_datalabels_path, test_labels, test_artifact_label, seconds, fs, initial_time, selected_channels)\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:23.873486Z","iopub.execute_input":"2022-09-16T20:42:23.873903Z","iopub.status.idle":"2022-09-16T20:42:30.979621Z","shell.execute_reply.started":"2022-09-16T20:42:23.873866Z","shell.execute_reply":"2022-09-16T20:42:30.978233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eegData: 3D np array [epochs x chans x ms]\nprint(\"=\"*20)\nprint(\"Train data shape:\",train_eegData.shape)\nprint(\"Test data shape:\",test_eegData.shape)\n\nprint(\"Artifacts in train data:\",sum(train_artifacts) / len(train_artifacts))\nprint(\"Artifacts in test data:\",sum(test_artifacts) / len(test_artifacts))","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:30.982741Z","iopub.execute_input":"2022-09-16T20:42:30.983198Z","iopub.status.idle":"2022-09-16T20:42:30.990103Z","shell.execute_reply.started":"2022-09-16T20:42:30.983155Z","shell.execute_reply":"2022-09-16T20:42:30.988881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge(data, labels):\n    assert len(data) == len(labels)\n    \n    new_data = data.reshape(-1, data.shape[2])\n    new_labels = [labels[0]] * data.shape[1]\n    for i in range(1, len(labels)):\n        new_data[data.shape[1]*i:data.shape[1]*(i+1)] = data[i, :, :]\n        new_labels += [labels[i]] * data.shape[1]\n\n    return np.array(new_data), np.array(new_labels)\n\ntrain_eegData, train_artifacts = merge(train_eegData, train_artifacts)\ntest_eegData, test_artifacts = merge(test_eegData, test_artifacts)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:30.991949Z","iopub.execute_input":"2022-09-16T20:42:30.992376Z","iopub.status.idle":"2022-09-16T20:42:31.192362Z","shell.execute_reply.started":"2022-09-16T20:42:30.992335Z","shell.execute_reply":"2022-09-16T20:42:31.191233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"=\"*20)\nprint(\"MERGING CHANNELS WITH EPOCHS\")\nprint(\"Train data new shape:\",train_eegData.shape)\nprint(\"Train data new labels shape:\",train_artifacts.shape)\n# [chans * epochs x ms]","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:31.193675Z","iopub.execute_input":"2022-09-16T20:42:31.194078Z","iopub.status.idle":"2022-09-16T20:42:31.200336Z","shell.execute_reply.started":"2022-09-16T20:42:31.194043Z","shell.execute_reply":"2022-09-16T20:42:31.199145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sum(train_artifacts) / len(train_artifacts)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:31.202078Z","iopub.execute_input":"2022-09-16T20:42:31.202470Z","iopub.status.idle":"2022-09-16T20:42:31.228302Z","shell.execute_reply.started":"2022-09-16T20:42:31.202418Z","shell.execute_reply":"2022-09-16T20:42:31.227500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"24.79% of this dataset is composed by artifact data","metadata":{}},{"cell_type":"code","source":"print(\"=\"*20)\nprint(f\"RESAMPLING THE DATA {fs} Hz ({seconds}s) to {new_fs} Hz\")\ntrain_eegData = signal.resample(train_eegData, int(new_fs*seconds), axis=1)\ntest_eegData = signal.resample(test_eegData, int(new_fs*seconds), axis=1)\nprint(\"Train data new shape:\",train_eegData.shape)\nprint(\"Train data new labels shape:\",train_artifacts.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:31.229748Z","iopub.execute_input":"2022-09-16T20:42:31.230108Z","iopub.status.idle":"2022-09-16T20:42:32.180569Z","shell.execute_reply.started":"2022-09-16T20:42:31.230077Z","shell.execute_reply":"2022-09-16T20:42:32.179500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training HBOS Model","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(train_eegData)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")\n\nprint(\"Labels length discovered by the model:\",len(clf.labels_))\n\nprint(\"Model Accuracy:\",sum(clf.labels_ == train_artifacts) / len(clf.labels_))\n\nevaluate_print(clf, train_artifacts, clf.labels_)\n\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(train_artifacts, clf.labels_, labels=[1,0]))\n\nprint(\"\\nCohen kappa:\",cohen_kappa_score(train_artifacts, clf.labels_))\nprint(\"F1 Score:\",f1_score(train_artifacts, clf.labels_))","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:32.182759Z","iopub.execute_input":"2022-09-16T20:42:32.183222Z","iopub.status.idle":"2022-09-16T20:42:36.724254Z","shell.execute_reply.started":"2022-09-16T20:42:32.183179Z","shell.execute_reply":"2022-09-16T20:42:36.722877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating HBOS Model","metadata":{}},{"cell_type":"code","source":"## EVALUATING THE MODEL ##\npred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nprint(\"Test Accuracy:\",sum(pred == test_artifacts) / len(pred))\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:42:36.728292Z","iopub.execute_input":"2022-09-16T20:42:36.728787Z","iopub.status.idle":"2022-09-16T20:42:38.384344Z","shell.execute_reply.started":"2022-09-16T20:42:36.728739Z","shell.execute_reply":"2022-09-16T20:42:38.383230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So this model isn't classifying randomly and it is actually good.","metadata":{}},{"cell_type":"code","source":"evaluate_print(clf, test_artifacts, pred)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:43:15.373035Z","iopub.execute_input":"2022-09-16T20:43:15.373483Z","iopub.status.idle":"2022-09-16T20:43:15.464501Z","shell.execute_reply.started":"2022-09-16T20:43:15.373440Z","shell.execute_reply":"2022-09-16T20:43:15.463065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\n\nfpr, tpr, thresholds = metrics.roc_curve(y_true=test_artifacts, y_score=pred)\nmetrics.auc(fpr, tpr)","metadata":{"execution":{"iopub.status.busy":"2022-09-16T20:45:15.923041Z","iopub.execute_input":"2022-09-16T20:45:15.923443Z","iopub.status.idle":"2022-09-16T20:45:15.942553Z","shell.execute_reply.started":"2022-09-16T20:45:15.923402Z","shell.execute_reply":"2022-09-16T20:45:15.941388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Saving the model","metadata":{}},{"cell_type":"code","source":"## SAVING THE MODEL\nif save_model:\n    print(\"=\"*20)\n    print(\"Saving the data...\")\n    filename = 'model.pickle'\n    pickle.dump(clf, open(filename, 'wb'))","metadata":{"execution":{"iopub.status.busy":"2022-09-06T22:35:46.465504Z","iopub.execute_input":"2022-09-06T22:35:46.466734Z","iopub.status.idle":"2022-09-06T22:35:46.476017Z","shell.execute_reply.started":"2022-09-06T22:35:46.466691Z","shell.execute_reply":"2022-09-06T22:35:46.474723Z"},"trusted":true},"execution_count":null,"outputs":[]}]}