{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%pip install pyod\nimport os\nimport numpy as np\nfrom pyod.models import hbos\nfrom pyod.utils.data import evaluate_print\nfrom sklearn.metrics import confusion_matrix,cohen_kappa_score,f1_score\nfrom scipy import signal\nimport matplotlib.pyplot as plt\nimport pickle\nimport pandas as pd\nfrom zipfile import ZipFile","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-30T17:22:43.752837Z","iopub.execute_input":"2022-09-30T17:22:43.753331Z","iopub.status.idle":"2022-09-30T17:23:02.480282Z","shell.execute_reply.started":"2022-09-30T17:22:43.753232Z","shell.execute_reply":"2022-09-30T17:23:02.479017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#####\n# Trial or epoch = eeg segment\n#####\n\nchannels = 32\nseconds = 0.300    # Size of each segment we want\nfs = 500.0         # Dtaset eeg sampling rate\ndata_portion = 0.4\nsave_model = False","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:02.483372Z","iopub.execute_input":"2022-09-30T17:23:02.483886Z","iopub.status.idle":"2022-09-30T17:23:02.490302Z","shell.execute_reply.started":"2022-09-30T17:23:02.483827Z","shell.execute_reply":"2022-09-30T17:23:02.489099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Extracting data\nZipFile(\"../input/grasp-and-lift-eeg-detection/train.zip\").extractall(\"./data\")\nZipFile(\"../input/grasp-and-lift-eeg-detection/test.zip\").extractall(\"./data\")","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:02.492103Z","iopub.execute_input":"2022-09-30T17:23:02.492651Z","iopub.status.idle":"2022-09-30T17:23:48.493673Z","shell.execute_reply.started":"2022-09-30T17:23:02.492607Z","shell.execute_reply":"2022-09-30T17:23:48.491310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('./data/train/subj1_series4_data.csv')[:1000]\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:48.498236Z","iopub.execute_input":"2022-09-30T17:23:48.498707Z","iopub.status.idle":"2022-09-30T17:23:49.256930Z","shell.execute_reply.started":"2022-09-30T17:23:48.498672Z","shell.execute_reply":"2022-09-30T17:23:49.255697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vizualizing some Data","metadata":{}},{"cell_type":"code","source":"plt.plot(df['Fp1'])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:49.259060Z","iopub.execute_input":"2022-09-30T17:23:49.259847Z","iopub.status.idle":"2022-09-30T17:23:49.507625Z","shell.execute_reply.started":"2022-09-30T17:23:49.259778Z","shell.execute_reply":"2022-09-30T17:23:49.506721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(df['Fp2'])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:49.509027Z","iopub.execute_input":"2022-09-30T17:23:49.509541Z","iopub.status.idle":"2022-09-30T17:23:49.702983Z","shell.execute_reply.started":"2022-09-30T17:23:49.509511Z","shell.execute_reply":"2022-09-30T17:23:49.701601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:49.704688Z","iopub.execute_input":"2022-09-30T17:23:49.705201Z","iopub.status.idle":"2022-09-30T17:23:49.732597Z","shell.execute_reply.started":"2022-09-30T17:23:49.705159Z","shell.execute_reply":"2022-09-30T17:23:49.731648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.drop('id', axis=1).shape","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:49.733984Z","iopub.execute_input":"2022-09-30T17:23:49.734524Z","iopub.status.idle":"2022-09-30T17:23:50.319889Z","shell.execute_reply.started":"2022-09-30T17:23:49.734493Z","shell.execute_reply":"2022-09-30T17:23:50.318576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"[fs * seconds x channels]","metadata":{}},{"cell_type":"code","source":"del df","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:50.321719Z","iopub.execute_input":"2022-09-30T17:23:50.322133Z","iopub.status.idle":"2022-09-30T17:23:50.330808Z","shell.execute_reply.started":"2022-09-30T17:23:50.322072Z","shell.execute_reply":"2022-09-30T17:23:50.329567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Dataset\n\nData notes:\n- sampling rate in 500 Hz\n- 32 Channels\n- We want segments of 300ms\n- 300ms in 500Hz is 150 points\n- data shape: [channels * segments x points]\n- We aren't going to use th","metadata":{}},{"cell_type":"code","source":"class EEGDataset2:\n    def __init__(self, datapath, seconds=0.300, fs=500, data_portion=0.7):\n        assert data_portion != 0.0, 'data_portion should be different from 0'\n        \n        self.datapath = datapath\n        self.recordings = os.listdir(datapath)\n        self.seconds = seconds\n        self.fs = fs\n        self.data_portion = data_portion\n    \n    def _get_data_fullpaths(self):\n        recordings_path = []\n        for recording in self.recordings:\n            recordings_path.append(os.path.join(self.datapath, recording))\n        return recordings_path\n    \n    def get_data(self):\n        recordings = []\n        data_fullpaths = self._get_data_fullpaths()\n        data_fullpaths_newlen = int(len(data_fullpaths) * data_portion)\n        data_fullpaths = data_fullpaths[:data_fullpaths_newlen]\n        \n        \n        for datapath in data_fullpaths:\n            df = pd.read_csv(datapath).drop('id', axis=1)\n            datapoints = int(self.seconds * self.fs)\n            df_len, channels = df.shape\n            for i in range(int(df_len / datapoints)):\n                for channel in range(channels):\n                    start = int(datapoints*i)\n                    end = int(datapoints * (i+1))\n                    recordings.append(df.to_numpy()[start:end, channel])\n                \n        return np.array(recordings)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:50.336387Z","iopub.execute_input":"2022-09-30T17:23:50.337061Z","iopub.status.idle":"2022-09-30T17:23:50.350938Z","shell.execute_reply.started":"2022-09-30T17:23:50.337022Z","shell.execute_reply":"2022-09-30T17:23:50.349761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datapath = './data/train'\ntest_datapath = './data/test'\n\nprint('Train data recordings:', len(os.listdir(train_datapath)))\nprint('Test data recordings:', len(os.listdir(test_datapath)))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:50.352570Z","iopub.execute_input":"2022-09-30T17:23:50.353415Z","iopub.status.idle":"2022-09-30T17:23:50.369915Z","shell.execute_reply.started":"2022-09-30T17:23:50.353369Z","shell.execute_reply":"2022-09-30T17:23:50.368599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset2 = EEGDataset2(train_datapath, seconds, fs, data_portion)\ntrain_data = dataset2.get_data()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:23:50.371515Z","iopub.execute_input":"2022-09-30T17:23:50.372374Z","iopub.status.idle":"2022-09-30T17:24:35.873194Z","shell.execute_reply.started":"2022-09-30T17:23:50.372324Z","shell.execute_reply":"2022-09-30T17:24:35.871914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:24:35.874990Z","iopub.execute_input":"2022-09-30T17:24:35.875414Z","iopub.status.idle":"2022-09-30T17:24:35.883439Z","shell.execute_reply.started":"2022-09-30T17:24:35.875377Z","shell.execute_reply":"2022-09-30T17:24:35.882281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training HBOS model","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(train_data)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:24:35.885432Z","iopub.execute_input":"2022-09-30T17:24:35.886107Z","iopub.status.idle":"2022-09-30T17:25:02.868355Z","shell.execute_reply.started":"2022-09-30T17:24:35.886072Z","shell.execute_reply":"2022-09-30T17:25:02.865975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_data[idx])\nclf.predict([train_data[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:02.870155Z","iopub.execute_input":"2022-09-30T17:25:02.871371Z","iopub.status.idle":"2022-09-30T17:25:03.073947Z","shell.execute_reply.started":"2022-09-30T17:25:02.871319Z","shell.execute_reply":"2022-09-30T17:25:03.072842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Another Dataset","metadata":{}},{"cell_type":"code","source":"channels = 32\nselected_channels = 32\ndataset_trial_duration = 1200 # Original dataset size of each trial (1.2 seconds)\nfs = 1000.0        # Original dataset eeg sampling rate\nseconds = 0.300    # Size of each segment we want\nnew_fs = 500.0     # Resize the dataset eeg sampling rate to 250.0 Hz\ninitial_time = 500 # We aren't going to pick the firsts 500ms because don't contain noise","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:03.075603Z","iopub.execute_input":"2022-09-30T17:25:03.076089Z","iopub.status.idle":"2022-09-30T17:25:03.083326Z","shell.execute_reply.started":"2022-09-30T17:25:03.076043Z","shell.execute_reply":"2022-09-30T17:25:03.081958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EEGDataset:\n    def __init__(self, data, datalabels, labels, artifact_label, seconds=0.3, fs=1000.0, new_fs=500.0, initial_time=200, channels=22):\n        self.data = np.load(data)\n        self.datalabels = np.load(datalabels)\n        \n        self.labels = labels\n        self.artifact_label = artifact_label\n        \n        self.seconds = seconds\n        self.fs = fs \n        self.duration = int(self.fs * self.seconds)\n        self.initial_time = initial_time\n        self.channels = channels\n        \n    def get_trials_from_channels(self):\n        trials = []\n        classes = []\n        artifacts = []\n        \n        for i in range(self.data.shape[2]):\n            class_ = self.datalabels[i]\n            if class_ not in self.labels:\n                continue\n            # [channels x duration, trials]\n            for channel in range(self.data.shape[0]):\n                trial = self.data[channel,self.initial_time:self.duration+self.initial_time,i]\n                trials.append(trial)\n                classes.append(class_)\n            \n                if class_ == self.artifact_label: artifacts.append(1)\n                else: artifacts.append(0)\n        \n        # Converting to np array\n        trials = np.array(trials)\n        \n        # Resampling to 500 Hz\n        trials = signal.resample(trials, int(new_fs*seconds), axis=1)\n                \n        return trials, classes, artifacts","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:03.085044Z","iopub.execute_input":"2022-09-30T17:25:03.085422Z","iopub.status.idle":"2022-09-30T17:25:03.103134Z","shell.execute_reply.started":"2022-09-30T17:25:03.085390Z","shell.execute_reply":"2022-09-30T17:25:03.101762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TRAIN DATASET ##\ntrain_data_path = '../input/eeg-data-from-passive-viewing/clr_EEG32.npy'\ntrain_datalabels_path = '../input/eeg-data-from-passive-viewing/clr_Labels.npy'\ntrain_labels = [7,8,9,10,11,12,13]\ntrain_artifact_label = 7\n\n## shape: [chans x ms x epochs]\n\ndataset = EEGDataset(train_data_path, train_datalabels_path, train_labels, train_artifact_label, seconds, fs, new_fs, initial_time, selected_channels)\ntrain_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:03.105239Z","iopub.execute_input":"2022-09-30T17:25:03.105678Z","iopub.status.idle":"2022-09-30T17:25:09.492481Z","shell.execute_reply.started":"2022-09-30T17:25:03.105636Z","shell.execute_reply":"2022-09-30T17:25:09.491009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TEST DATASET ##\ntest_data_path = '../input/eeg-data-from-passive-viewing/ori_EEG32.npy'\ntest_datalabels_path = '../input/eeg-data-from-passive-viewing/ori_Labels.npy'\ntest_labels = [1,2,3,4,5,6,14]\ntest_artifact_label = 14\n\n\ntest_dataset = EEGDataset(test_data_path, test_datalabels_path, test_labels, test_artifact_label, seconds, fs, new_fs, initial_time, selected_channels)\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:09.494276Z","iopub.execute_input":"2022-09-30T17:25:09.494687Z","iopub.status.idle":"2022-09-30T17:25:15.601101Z","shell.execute_reply.started":"2022-09-30T17:25:09.494656Z","shell.execute_reply":"2022-09-30T17:25:15.599742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_eegData.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:15.603029Z","iopub.execute_input":"2022-09-30T17:25:15.603489Z","iopub.status.idle":"2022-09-30T17:25:15.612138Z","shell.execute_reply.started":"2022-09-30T17:25:15.603446Z","shell.execute_reply":"2022-09-30T17:25:15.610827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_eegData.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:15.613765Z","iopub.execute_input":"2022-09-30T17:25:15.614610Z","iopub.status.idle":"2022-09-30T17:25:15.625566Z","shell.execute_reply.started":"2022-09-30T17:25:15.614566Z","shell.execute_reply":"2022-09-30T17:25:15.624229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(train_eegData[10])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:15.627307Z","iopub.execute_input":"2022-09-30T17:25:15.627766Z","iopub.status.idle":"2022-09-30T17:25:15.856614Z","shell.execute_reply.started":"2022-09-30T17:25:15.627725Z","shell.execute_reply":"2022-09-30T17:25:15.855633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(train_eegData)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")\n\nprint(\"Labels length discovered by the model:\",len(clf.labels_))\n\nprint(\"Model Accuracy:\",sum(clf.labels_ == train_artifacts) / len(clf.labels_))\n\nevaluate_print(clf, train_artifacts, clf.labels_)\n\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(train_artifacts, clf.labels_, labels=[1,0]))\n\nprint(\"\\nCohen kappa:\",cohen_kappa_score(train_artifacts, clf.labels_))\nprint(\"F1 Score:\",f1_score(train_artifacts, clf.labels_))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:15.858529Z","iopub.execute_input":"2022-09-30T17:25:15.859295Z","iopub.status.idle":"2022-09-30T17:25:19.630247Z","shell.execute_reply.started":"2022-09-30T17:25:15.859250Z","shell.execute_reply":"2022-09-30T17:25:19.628892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluating the model","metadata":{}},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:19.631760Z","iopub.execute_input":"2022-09-30T17:25:19.632140Z","iopub.status.idle":"2022-09-30T17:25:21.154107Z","shell.execute_reply.started":"2022-09-30T17:25:19.632108Z","shell.execute_reply":"2022-09-30T17:25:21.152863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adjusting to both datasets","metadata":{}},{"cell_type":"code","source":"idx = 1000\nplt.plot(test_eegData[idx])\nclf.predict([test_eegData[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:21.155527Z","iopub.execute_input":"2022-09-30T17:25:21.155920Z","iopub.status.idle":"2022-09-30T17:25:21.335294Z","shell.execute_reply.started":"2022-09-30T17:25:21.155886Z","shell.execute_reply":"2022-09-30T17:25:21.333882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_data[idx])\nclf.predict([train_data[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:21.337084Z","iopub.execute_input":"2022-09-30T17:25:21.341216Z","iopub.status.idle":"2022-09-30T17:25:21.537739Z","shell.execute_reply.started":"2022-09-30T17:25:21.341168Z","shell.execute_reply":"2022-09-30T17:25:21.536219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalizing data","metadata":{}},{"cell_type":"code","source":"mean = train_data.mean()\nstd = train_data.std()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:21.539549Z","iopub.execute_input":"2022-09-30T17:25:21.540000Z","iopub.status.idle":"2022-09-30T17:25:27.052396Z","shell.execute_reply.started":"2022-09-30T17:25:21.539963Z","shell.execute_reply":"2022-09-30T17:25:27.051321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = (train_data - mean) / std\ntest_eegData = (test_eegData - mean) / std\ntrain_eegData = (train_eegData - mean) / std","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:27.059495Z","iopub.execute_input":"2022-09-30T17:25:27.060274Z","iopub.status.idle":"2022-09-30T17:25:28.438664Z","shell.execute_reply.started":"2022-09-30T17:25:27.060230Z","shell.execute_reply":"2022-09-30T17:25:28.437430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retraining the model with only one dataset","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(train_data)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:28.440417Z","iopub.execute_input":"2022-09-30T17:25:28.441292Z","iopub.status.idle":"2022-09-30T17:25:56.299979Z","shell.execute_reply.started":"2022-09-30T17:25:28.441248Z","shell.execute_reply":"2022-09-30T17:25:56.298352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_data[idx])\nclf.predict([train_data[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:56.301982Z","iopub.execute_input":"2022-09-30T17:25:56.303075Z","iopub.status.idle":"2022-09-30T17:25:56.513846Z","shell.execute_reply.started":"2022-09-30T17:25:56.303025Z","shell.execute_reply":"2022-09-30T17:25:56.512596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_eegData[idx])\nclf.predict([train_eegData[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:56.515290Z","iopub.execute_input":"2022-09-30T17:25:56.516209Z","iopub.status.idle":"2022-09-30T17:25:56.716638Z","shell.execute_reply.started":"2022-09-30T17:25:56.516166Z","shell.execute_reply":"2022-09-30T17:25:56.715190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(test_eegData[idx])\nclf.predict([test_eegData[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:56.718292Z","iopub.execute_input":"2022-09-30T17:25:56.718633Z","iopub.status.idle":"2022-09-30T17:25:56.930055Z","shell.execute_reply.started":"2022-09-30T17:25:56.718604Z","shell.execute_reply":"2022-09-30T17:25:56.928606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluating witn another dataset","metadata":{}},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:56.931650Z","iopub.execute_input":"2022-09-30T17:25:56.932646Z","iopub.status.idle":"2022-09-30T17:25:58.597222Z","shell.execute_reply.started":"2022-09-30T17:25:56.932607Z","shell.execute_reply":"2022-09-30T17:25:58.595897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"maybe the model is inverting the results, so maybe if we invert the predictions, we should get better accuracy","metadata":{}},{"cell_type":"code","source":"pred = np.where((pred==0)|(pred==1), pred^1, pred)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:58.599072Z","iopub.execute_input":"2022-09-30T17:25:58.599461Z","iopub.status.idle":"2022-09-30T17:25:59.035538Z","shell.execute_reply.started":"2022-09-30T17:25:58.599429Z","shell.execute_reply":"2022-09-30T17:25:59.034261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"No, the model is shit","metadata":{}},{"cell_type":"markdown","source":"## Normalizing the datasets with their mean and std","metadata":{}},{"cell_type":"markdown","source":"### Realoding data","metadata":{}},{"cell_type":"code","source":"train_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:25:59.037768Z","iopub.execute_input":"2022-09-30T17:25:59.038736Z","iopub.status.idle":"2022-09-30T17:26:01.228983Z","shell.execute_reply.started":"2022-09-30T17:25:59.038683Z","shell.execute_reply":"2022-09-30T17:26:01.227617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_eegData[idx])\nclf.predict([train_eegData[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:01.230669Z","iopub.execute_input":"2022-09-30T17:26:01.231190Z","iopub.status.idle":"2022-09-30T17:26:01.443539Z","shell.execute_reply.started":"2022-09-30T17:26:01.231144Z","shell.execute_reply":"2022-09-30T17:26:01.442276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(train_data[idx])\nclf.predict([train_data[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:01.445207Z","iopub.execute_input":"2022-09-30T17:26:01.446332Z","iopub.status.idle":"2022-09-30T17:26:01.655210Z","shell.execute_reply.started":"2022-09-30T17:26:01.446282Z","shell.execute_reply":"2022-09-30T17:26:01.653621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalizing","metadata":{}},{"cell_type":"code","source":"eegDataMean = train_eegData.mean()\neegDataStd = train_eegData.std()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:01.657063Z","iopub.execute_input":"2022-09-30T17:26:01.658264Z","iopub.status.idle":"2022-09-30T17:26:01.728157Z","shell.execute_reply.started":"2022-09-30T17:26:01.658215Z","shell.execute_reply":"2022-09-30T17:26:01.726728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_eegData = (train_eegData - eegDataMean) / eegDataStd\ntest_eegData = (test_eegData - eegDataMean) / eegDataStd","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:01.732938Z","iopub.execute_input":"2022-09-30T17:26:01.734437Z","iopub.status.idle":"2022-09-30T17:26:01.812489Z","shell.execute_reply.started":"2022-09-30T17:26:01.734385Z","shell.execute_reply":"2022-09-30T17:26:01.811214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 4000\nplt.plot(test_eegData[idx])\nclf.predict([test_eegData[idx]])","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:01.813996Z","iopub.execute_input":"2022-09-30T17:26:01.814396Z","iopub.status.idle":"2022-09-30T17:26:02.027198Z","shell.execute_reply.started":"2022-09-30T17:26:01.814362Z","shell.execute_reply":"2022-09-30T17:26:02.025819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reavaluating the model","metadata":{}},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:02.028663Z","iopub.execute_input":"2022-09-30T17:26:02.029068Z","iopub.status.idle":"2022-09-30T17:26:03.626343Z","shell.execute_reply.started":"2022-09-30T17:26:02.029030Z","shell.execute_reply":"2022-09-30T17:26:03.624750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This model actually  is way better training in another dataset\n\nAs the model is classifying good the labeled dataset, we can conclude that it is classifying equally the other non labeled dataset","metadata":{}},{"cell_type":"markdown","source":"## Reloading the dataset and don't normalizing it","metadata":{}},{"cell_type":"code","source":"train_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:03.628082Z","iopub.execute_input":"2022-09-30T17:26:03.628582Z","iopub.status.idle":"2022-09-30T17:26:05.813237Z","shell.execute_reply.started":"2022-09-30T17:26:03.628550Z","shell.execute_reply":"2022-09-30T17:26:05.812085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:05.814566Z","iopub.execute_input":"2022-09-30T17:26:05.815069Z","iopub.status.idle":"2022-09-30T17:26:08.028184Z","shell.execute_reply.started":"2022-09-30T17:26:05.815025Z","shell.execute_reply":"2022-09-30T17:26:08.027218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we can conclude that normalize the dataset with their mean was very important.\n\nTo pass data to our model, first we will have to analyse it scale and normalize with the mean of one or another dataset used here.","metadata":{}},{"cell_type":"markdown","source":"## Manually Evaluating the model by eye","metadata":{}},{"cell_type":"markdown","source":"To cofirm that the model is performing good in the unlabeled we need to check it manually","metadata":{}},{"cell_type":"code","source":"for idx in range(30):\n    pred = clf.predict([train_data[idx]])\n    print(\"Prediction:\",pred)\n    \n    plt.plot(train_data[idx], color='r' if pred == 1 else 'b')\n    plt.show()\n    print(\"=\"*20)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:08.029447Z","iopub.execute_input":"2022-09-30T17:26:08.030157Z","iopub.status.idle":"2022-09-30T17:26:13.637885Z","shell.execute_reply.started":"2022-09-30T17:26:08.030121Z","shell.execute_reply":"2022-09-30T17:26:13.636414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looks like the model is evaluating ok, in this case we cannot evaluate automatically, because one dataset doesn't contain labels","metadata":{}},{"cell_type":"markdown","source":"# Training the model with both datasets\n\n\nWe are going to train the model with both datasets","metadata":{}},{"cell_type":"markdown","source":"## Reloading the datasets and normalizing it","metadata":{}},{"cell_type":"code","source":"train_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()\ntrain_data = dataset2.get_data()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:13.639502Z","iopub.execute_input":"2022-09-30T17:26:13.640591Z","iopub.status.idle":"2022-09-30T17:26:57.849854Z","shell.execute_reply.started":"2022-09-30T17:26:13.640542Z","shell.execute_reply":"2022-09-30T17:26:57.848421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = train_data.mean()\nstd = train_data.std()\n\neegDataMean = train_eegData.mean()\neegDataStd = train_eegData.std()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:26:57.851581Z","iopub.execute_input":"2022-09-30T17:26:57.851996Z","iopub.status.idle":"2022-09-30T17:27:00.807251Z","shell.execute_reply.started":"2022-09-30T17:26:57.851959Z","shell.execute_reply":"2022-09-30T17:27:00.805967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = (train_data - mean) / std\n\ntrain_eegData = (train_eegData - eegDataMean) / eegDataStd\ntest_eegData = (test_eegData - eegDataMean) / eegDataStd","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:00.808574Z","iopub.execute_input":"2022-09-30T17:27:00.808942Z","iopub.status.idle":"2022-09-30T17:27:02.673702Z","shell.execute_reply.started":"2022-09-30T17:27:00.808912Z","shell.execute_reply":"2022-09-30T17:27:02.672393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Joining both datasets","metadata":{}},{"cell_type":"code","source":"merged_dataset = np.concatenate([train_data, train_eegData], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:02.675424Z","iopub.execute_input":"2022-09-30T17:27:02.676609Z","iopub.status.idle":"2022-09-30T17:27:05.161768Z","shell.execute_reply.started":"2022-09-30T17:27:02.676561Z","shell.execute_reply":"2022-09-30T17:27:05.160237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_dataset.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:05.163508Z","iopub.execute_input":"2022-09-30T17:27:05.163927Z","iopub.status.idle":"2022-09-30T17:27:05.171857Z","shell.execute_reply.started":"2022-09-30T17:27:05.163890Z","shell.execute_reply":"2022-09-30T17:27:05.170532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data, train_eegData","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:05.173678Z","iopub.execute_input":"2022-09-30T17:27:05.174358Z","iopub.status.idle":"2022-09-30T17:27:05.195380Z","shell.execute_reply.started":"2022-09-30T17:27:05.174312Z","shell.execute_reply":"2022-09-30T17:27:05.194250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retraining the model with both datasets","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(merged_dataset)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:05.196883Z","iopub.execute_input":"2022-09-30T17:27:05.197272Z","iopub.status.idle":"2022-09-30T17:27:35.488114Z","shell.execute_reply.started":"2022-09-30T17:27:05.197237Z","shell.execute_reply":"2022-09-30T17:27:35.486812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluating","metadata":{}},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:35.489459Z","iopub.execute_input":"2022-09-30T17:27:35.489863Z","iopub.status.idle":"2022-09-30T17:27:37.047453Z","shell.execute_reply.started":"2022-09-30T17:27:35.489824Z","shell.execute_reply":"2022-09-30T17:27:37.046045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The model is just Ok","metadata":{}},{"cell_type":"code","source":"for idx in range(30):\n    pred = clf.predict([merged_dataset[idx]])\n    print(\"Prediction:\",pred)\n    \n    plt.plot(merged_dataset[idx], color='r' if pred == 1 else 'b')\n    plt.show()\n    print(\"=\"*20)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:37.049564Z","iopub.execute_input":"2022-09-30T17:27:37.050258Z","iopub.status.idle":"2022-09-30T17:27:41.808883Z","shell.execute_reply.started":"2022-09-30T17:27:37.050206Z","shell.execute_reply":"2022-09-30T17:27:41.807880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retraining the model with differents aproaches","metadata":{}},{"cell_type":"markdown","source":"We are going to normalize the data this time","metadata":{}},{"cell_type":"code","source":"train_eegData, train_classes, train_artifacts = dataset.get_trials_from_channels()\ntest_eegData, test_classes, test_artifacts = test_dataset.get_trials_from_channels()\ntrain_data = dataset2.get_data()","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:27:41.810328Z","iopub.execute_input":"2022-09-30T17:27:41.810899Z","iopub.status.idle":"2022-09-30T17:28:31.159035Z","shell.execute_reply.started":"2022-09-30T17:27:41.810860Z","shell.execute_reply":"2022-09-30T17:28:31.157373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_dataset = np.concatenate([train_data, train_eegData], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:28:31.161701Z","iopub.execute_input":"2022-09-30T17:28:31.162738Z","iopub.status.idle":"2022-09-30T17:28:32.804760Z","shell.execute_reply.started":"2022-09-30T17:28:31.162678Z","shell.execute_reply":"2022-09-30T17:28:32.803473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data, train_eegData","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:28:32.806138Z","iopub.execute_input":"2022-09-30T17:28:32.806511Z","iopub.status.idle":"2022-09-30T17:28:32.818112Z","shell.execute_reply.started":"2022-09-30T17:28:32.806480Z","shell.execute_reply":"2022-09-30T17:28:32.816547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Retraining","metadata":{}},{"cell_type":"code","source":"clf = hbos.HBOS(n_bins=17, alpha=0.07, tol=0.5,contamination=.15)\nclf.fit(merged_dataset)\n\nprint(\"=\"*20)\nprint(\"TRAINING THE MODEL\")","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:28:32.819973Z","iopub.execute_input":"2022-09-30T17:28:32.820938Z","iopub.status.idle":"2022-09-30T17:29:01.881186Z","shell.execute_reply.started":"2022-09-30T17:28:32.820886Z","shell.execute_reply":"2022-09-30T17:29:01.879548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Evaluating","metadata":{}},{"cell_type":"code","source":"pred = clf.predict(test_eegData)\n\nprint(\"=\"*20)\nprint(\"EVALUATING THE MODEL\")\nevaluate_print(clf, test_artifacts, pred)\nprint(\"\\nConfusion Matrix\")\nprint(confusion_matrix(test_artifacts, pred, labels=[1,0]))\nprint(\"Test Cohen Kappa:\",cohen_kappa_score(test_artifacts, pred))\nprint(\"Test F1 Score:\",f1_score(test_artifacts, pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-30T17:29:01.882811Z","iopub.execute_input":"2022-09-30T17:29:01.883296Z","iopub.status.idle":"2022-09-30T17:29:03.403960Z","shell.execute_reply.started":"2022-09-30T17:29:01.883250Z","shell.execute_reply":"2022-09-30T17:29:03.402541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mean and std selector to normalize some signal","metadata":{}},{"cell_type":"markdown","source":"We can do this by measuering the distance of the data mean and std of the saved data mean and std\n\nWe could also use a machine learning to classify wich mean and std we are going to use to normalize some data\n\nAfter done, we are going to test if this is working","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}