I'm currently trying to contribute this Dataset class to pytorch-lightning-bolts. It converts the csv files into binary, resulting in significant performance gains.
\nThis code is public domain and you should feel free to use it. I am paying for the hosting right now, so please be courteous :)
\nimport os\nimport time\nimport zipfile\n\nimport requests\nimport torch\nimport torch.utils.data as data\n\n\ndef recursive_listdir(path: str, suffix: str) -> list:\n return [os.path.join(dp, f)\n for dp, dn, fn in os.walk(os.path.expanduser(path))\n for f in fn\n if f.endswith(suffix)]\n\n\nclass GraspAndLiftEEGDataset(data.Dataset):\n \"\"\" 32-channel, 500Hz EEG dataset of subjects performing a various\n grasp-and-lift motor tasks, with per-sample class labels.\n\n There are 12 subjects in total, 10 series of trials for each subject,\n and approximately 30 trials within each series. The number of trials\n varies for each series. The training set contains the first 8 series\n for each subject. The test set contains the 9th and 10th series.\n\n Videos of trials being performed:\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM69_ESM.avi\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM70_ESM.avi\n (Note: these videos are also available in the S3 bucket)\n\n Args:\n root: Path to directory containing train/ and test/ folders\n\n train: If true, use the train/ directory and load class labels.\n If false, load the test/ directory (which lacks labels)\n\n download: If true, download the data if it is not present locally\n\n num_samples: The number of samples in the returned examples.\n If None, the the dataset yields the full length trials.\n\n last_label_only: If true, return only the last sample's labels.\n Must be used with num_samples.\n\n Examples:\n >>> dataset = GraspAndLiftEEGDataset('/data', train=True, download=True, num_samples=1024)\n Downloading from https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip\n Downloaded in 283 seconds\n Extracting /data/grasp-and-lift-eeg-detection.zip to /data\n Unzipped in 36 seconds\n >>> len(dataset)\n 17887546\n >>> data, label = dataset[0]\n >>> data.shape\n torch.Size([32, 1024])\n >>> label.shape\n torch.Size([6, 1024])\n\n Labels:\n 1. HandStart\n 2. FirstDigitTouch\n 3. BothStartLoadPhase\n 4. LiftOff\n 5. Replace\n 6. BothReleased\n\n Reference:\n Luciw, M., Jarocka, E. & Edin, B. Multi-channel EEG recordings during 3,936\n grasp and lift trials with varying weight and friction. Sci Data 1, 140047\n (2014). https://doi.org/10.1038/sdata.2014.47\n\n https://www.kaggle.com/c/grasp-and-lift-eeg-detection\n\n License:\n Original data has been made available under the terms of Attribution 4.0\n International Creative Commons License (http://creativecommons.org/licenses/by/4.0/).\n \"\"\"\n\n GRASPLIFT_DATA_HEADER = 'id,Fp1,Fp2,F7,F3,Fz,F4,F8,FC5,FC1,FC2,FC6,T7,C3,Cz,C4,T8,TP9,CP5,CP1,CP2,CP6,' + \\\n 'TP10,P7,P3,Pz,P4,P8,PO9,O1,Oz,O2,PO10\\n'\n\n GRASPLIFT_EVENTS_HEADER = 'id,HandStart,FirstDigitTouch,BothStartLoadPhase,LiftOff,Replace,BothReleased\\n'\n\n ZIP_URL = 'https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip'\n\n ZIP_SIZE_BYTES = 980887394\n\n def __init__(self,\n root: str,\n train: bool = True,\n download: bool = True,\n num_samples: int = None,\n last_label_only: bool = False):\n super(GraspAndLiftEEGDataset, self).__init__()\n if num_samples is None and last_label_only:\n raise ValueError('last_label_only cannot be used without setting num_samples')\n self.num_samples = num_samples\n self.last_label_only = last_label_only\n data_dir = os.path.join(root, 'train' if train else 'test')\n if not os.path.exists(data_dir):\n if not download:\n raise ValueError(f'{data_dir} does not exist')\n if not os.path.exists(root):\n os.makedirs(root)\n self.download(root)\n csv_suffix = '.csv'\n bin_suffix = '.csv.bin'\n csv_files = recursive_listdir(data_dir, csv_suffix)\n bin_files = recursive_listdir(data_dir, bin_suffix)\n should_compile = len(bin_files) < len(csv_files)\n if should_compile:\n print(f'Number of .csv.bin files ({len(bin_files)}) '\n f'is less than the number of .csv ({len(csv_files)}).'\n ' Compiling binary representation...')\n self.load_from_csv(csv_files)\n else:\n self.load_from_bin(bin_files)\n\n def load_from_csv(self, csv_files):\n self.X, self.Y = self.compile_bin(csv_files)\n if self.num_samples is not None:\n # Divide each example up into windows\n self.total_examples = 0\n for x in self.X:\n self.total_examples += x.shape[1] - self.num_samples + 1\n\n def load_from_bin(self, bin_files: list):\n examples = {}\n self.total_examples = 0\n for file in bin_files:\n is_data = file.endswith('_data.csv.bin')\n series = file[:-len('_data.csv.bin')\n if is_data else -len('_events.csv.bin')]\n samples = torch.load(file)\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n if is_data and self.num_samples is not None:\n self.total_examples += samples.shape[1] - self.num_samples + 1\n self.X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n self.X.append(x)\n if y is not None:\n Y.append(y)\n self.Y = Y if len(Y) > 0 else None\n\n def download(self, root: str):\n zip_path = os.path.join(root, 'grasp-and-lift-eeg-detection.zip')\n if not os.path.exists(zip_path) or os.path.getsize(zip_path) != self.ZIP_SIZE_BYTES:\n print(f'Downloading from {self.ZIP_URL}')\n start = time.time()\n r = requests.get(self.ZIP_URL)\n if r.status_code != 200:\n raise ValueError(\n f'Expected status code 200, got {r.status_code}')\n with open(zip_path, 'wb') as f:\n f.write(r.content)\n delta = time.time() - start\n print(f'Downloaded in {int(delta)} seconds')\n print(f'Extracting {zip_path} to {root}')\n start = time.time()\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(root)\n delta = time.time() - start\n print(f'Unzipped in {int(delta)} seconds')\n os.remove(zip_path)\n\n def compile_bin(self, csv_files: list):\n examples = {}\n for i, file in enumerate(csv_files):\n is_data = file.endswith('_data.csv')\n samples = []\n with open(file, 'r') as f:\n hdr = f.readline()\n expected_hdr = self.GRASPLIFT_DATA_HEADER if is_data else self.GRASPLIFT_EVENTS_HEADER\n if hdr != expected_hdr:\n raise ValueError('bad header')\n for line in f:\n channels = line.strip().split(',')[1:]\n if is_data:\n # Data is converted to float eventually anyway\n channels = [float(x) for x in channels]\n else:\n # Labels are integer format\n channels = [int(x) for x in channels]\n channels = torch.Tensor(channels).unsqueeze(1)\n samples.append(channels)\n samples = torch.cat(samples, dim=1)\n series = file[:-len('_data.csv')\n if is_data else -len('_events.csv')]\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n print(f'Processed {i+1}/{len(csv_files)} {file}')\n X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n torch.save(samples, series + '_data.csv.bin')\n X.append(x)\n if y is not None:\n torch.save(samples, series + '_events.csv.bin')\n Y.append(y)\n return X, Y if len(Y) > 0 else None\n\n def __getitem__(self, index):\n if self.num_samples is None:\n # Return the entire example (e.g. reinforcement learning)\n return (self.X[index], self.Y[index] if self.Y is not None else [])\n # Find the example and offset for the index\n ofs = 0\n for i, x in enumerate(self.X):\n num_examples = x.shape[1] - self.num_samples + 1\n if index >= ofs + num_examples:\n ofs += num_examples\n continue\n j = index - ofs\n x = x[:, j:j + self.num_samples]\n if self.Y is not None:\n if self.last_label_only:\n # Only return the label for the last sample\n y = self.Y[i][:, j + self.num_samples - 1]\n else:\n # Return labels for all samples\n y = self.Y[i][:, j:j + self.num_samples]\n else:\n y = []\n return x, y\n raise ValueError(f'unable to seek {index}')\n\n def __len__(self):\n if self.num_samples is None:\n # No windowing - each example is full length\n return len(self.X)\n # Use precalculated dataset length\n return self.total_examples\nThe latest version (if updated) is available at: https://github.com/thavlik/machine-learning-portfolio/blob/master/src/dataset/grasp_and_lift_eeg.py
\nBig thanks to WAY Consortium for making this dataset freely available!
", "messages": [ { "id": 1113357, "postDate": "2020-12-15T11:47:12.987Z", "content": "I'm currently trying to contribute this Dataset class to pytorch-lightning-bolts. It converts the csv files into binary, resulting in significant performance gains.
\nThis code is public domain and you should feel free to use it. I am paying for the hosting right now, so please be courteous :)
\nimport os\nimport time\nimport zipfile\n\nimport requests\nimport torch\nimport torch.utils.data as data\n\n\ndef recursive_listdir(path: str, suffix: str) -> list:\n return [os.path.join(dp, f)\n for dp, dn, fn in os.walk(os.path.expanduser(path))\n for f in fn\n if f.endswith(suffix)]\n\n\nclass GraspAndLiftEEGDataset(data.Dataset):\n \"\"\" 32-channel, 500Hz EEG dataset of subjects performing a various\n grasp-and-lift motor tasks, with per-sample class labels.\n\n There are 12 subjects in total, 10 series of trials for each subject,\n and approximately 30 trials within each series. The number of trials\n varies for each series. The training set contains the first 8 series\n for each subject. The test set contains the 9th and 10th series.\n\n Videos of trials being performed:\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM69_ESM.avi\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM70_ESM.avi\n (Note: these videos are also available in the S3 bucket)\n\n Args:\n root: Path to directory containing train/ and test/ folders\n\n train: If true, use the train/ directory and load class labels.\n If false, load the test/ directory (which lacks labels)\n\n download: If true, download the data if it is not present locally\n\n num_samples: The number of samples in the returned examples.\n If None, the the dataset yields the full length trials.\n\n last_label_only: If true, return only the last sample's labels.\n Must be used with num_samples.\n\n Examples:\n >>> dataset = GraspAndLiftEEGDataset('/data', train=True, download=True, num_samples=1024)\n Downloading from https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip\n Downloaded in 283 seconds\n Extracting /data/grasp-and-lift-eeg-detection.zip to /data\n Unzipped in 36 seconds\n >>> len(dataset)\n 17887546\n >>> data, label = dataset[0]\n >>> data.shape\n torch.Size([32, 1024])\n >>> label.shape\n torch.Size([6, 1024])\n\n Labels:\n 1. HandStart\n 2. FirstDigitTouch\n 3. BothStartLoadPhase\n 4. LiftOff\n 5. Replace\n 6. BothReleased\n\n Reference:\n Luciw, M., Jarocka, E. & Edin, B. Multi-channel EEG recordings during 3,936\n grasp and lift trials with varying weight and friction. Sci Data 1, 140047\n (2014). https://doi.org/10.1038/sdata.2014.47\n\n https://www.kaggle.com/c/grasp-and-lift-eeg-detection\n\n License:\n Original data has been made available under the terms of Attribution 4.0\n International Creative Commons License (http://creativecommons.org/licenses/by/4.0/).\n \"\"\"\n\n GRASPLIFT_DATA_HEADER = 'id,Fp1,Fp2,F7,F3,Fz,F4,F8,FC5,FC1,FC2,FC6,T7,C3,Cz,C4,T8,TP9,CP5,CP1,CP2,CP6,' + \\\n 'TP10,P7,P3,Pz,P4,P8,PO9,O1,Oz,O2,PO10\\n'\n\n GRASPLIFT_EVENTS_HEADER = 'id,HandStart,FirstDigitTouch,BothStartLoadPhase,LiftOff,Replace,BothReleased\\n'\n\n ZIP_URL = 'https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip'\n\n ZIP_SIZE_BYTES = 980887394\n\n def __init__(self,\n root: str,\n train: bool = True,\n download: bool = True,\n num_samples: int = None,\n last_label_only: bool = False):\n super(GraspAndLiftEEGDataset, self).__init__()\n if num_samples is None and last_label_only:\n raise ValueError('last_label_only cannot be used without setting num_samples')\n self.num_samples = num_samples\n self.last_label_only = last_label_only\n data_dir = os.path.join(root, 'train' if train else 'test')\n if not os.path.exists(data_dir):\n if not download:\n raise ValueError(f'{data_dir} does not exist')\n if not os.path.exists(root):\n os.makedirs(root)\n self.download(root)\n csv_suffix = '.csv'\n bin_suffix = '.csv.bin'\n csv_files = recursive_listdir(data_dir, csv_suffix)\n bin_files = recursive_listdir(data_dir, bin_suffix)\n should_compile = len(bin_files) < len(csv_files)\n if should_compile:\n print(f'Number of .csv.bin files ({len(bin_files)}) '\n f'is less than the number of .csv ({len(csv_files)}).'\n ' Compiling binary representation...')\n self.load_from_csv(csv_files)\n else:\n self.load_from_bin(bin_files)\n\n def load_from_csv(self, csv_files):\n self.X, self.Y = self.compile_bin(csv_files)\n if self.num_samples is not None:\n # Divide each example up into windows\n self.total_examples = 0\n for x in self.X:\n self.total_examples += x.shape[1] - self.num_samples + 1\n\n def load_from_bin(self, bin_files: list):\n examples = {}\n self.total_examples = 0\n for file in bin_files:\n is_data = file.endswith('_data.csv.bin')\n series = file[:-len('_data.csv.bin')\n if is_data else -len('_events.csv.bin')]\n samples = torch.load(file)\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n if is_data and self.num_samples is not None:\n self.total_examples += samples.shape[1] - self.num_samples + 1\n self.X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n self.X.append(x)\n if y is not None:\n Y.append(y)\n self.Y = Y if len(Y) > 0 else None\n\n def download(self, root: str):\n zip_path = os.path.join(root, 'grasp-and-lift-eeg-detection.zip')\n if not os.path.exists(zip_path) or os.path.getsize(zip_path) != self.ZIP_SIZE_BYTES:\n print(f'Downloading from {self.ZIP_URL}')\n start = time.time()\n r = requests.get(self.ZIP_URL)\n if r.status_code != 200:\n raise ValueError(\n f'Expected status code 200, got {r.status_code}')\n with open(zip_path, 'wb') as f:\n f.write(r.content)\n delta = time.time() - start\n print(f'Downloaded in {int(delta)} seconds')\n print(f'Extracting {zip_path} to {root}')\n start = time.time()\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(root)\n delta = time.time() - start\n print(f'Unzipped in {int(delta)} seconds')\n os.remove(zip_path)\n\n def compile_bin(self, csv_files: list):\n examples = {}\n for i, file in enumerate(csv_files):\n is_data = file.endswith('_data.csv')\n samples = []\n with open(file, 'r') as f:\n hdr = f.readline()\n expected_hdr = self.GRASPLIFT_DATA_HEADER if is_data else self.GRASPLIFT_EVENTS_HEADER\n if hdr != expected_hdr:\n raise ValueError('bad header')\n for line in f:\n channels = line.strip().split(',')[1:]\n if is_data:\n # Data is converted to float eventually anyway\n channels = [float(x) for x in channels]\n else:\n # Labels are integer format\n channels = [int(x) for x in channels]\n channels = torch.Tensor(channels).unsqueeze(1)\n samples.append(channels)\n samples = torch.cat(samples, dim=1)\n series = file[:-len('_data.csv')\n if is_data else -len('_events.csv')]\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n print(f'Processed {i+1}/{len(csv_files)} {file}')\n X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n torch.save(samples, series + '_data.csv.bin')\n X.append(x)\n if y is not None:\n torch.save(samples, series + '_events.csv.bin')\n Y.append(y)\n return X, Y if len(Y) > 0 else None\n\n def __getitem__(self, index):\n if self.num_samples is None:\n # Return the entire example (e.g. reinforcement learning)\n return (self.X[index], self.Y[index] if self.Y is not None else [])\n # Find the example and offset for the index\n ofs = 0\n for i, x in enumerate(self.X):\n num_examples = x.shape[1] - self.num_samples + 1\n if index >= ofs + num_examples:\n ofs += num_examples\n continue\n j = index - ofs\n x = x[:, j:j + self.num_samples]\n if self.Y is not None:\n if self.last_label_only:\n # Only return the label for the last sample\n y = self.Y[i][:, j + self.num_samples - 1]\n else:\n # Return labels for all samples\n y = self.Y[i][:, j:j + self.num_samples]\n else:\n y = []\n return x, y\n raise ValueError(f'unable to seek {index}')\n\n def __len__(self):\n if self.num_samples is None:\n # No windowing - each example is full length\n return len(self.X)\n # Use precalculated dataset length\n return self.total_examples\nThe latest version (if updated) is available at: https://github.com/thavlik/machine-learning-portfolio/blob/master/src/dataset/grasp_and_lift_eeg.py
\nBig thanks to WAY Consortium for making this dataset freely available!
", "rawMarkdown": "I'm currently trying to contribute this Dataset class to pytorch-lightning-bolts. It converts the csv files into binary, resulting in significant performance gains.\n\nThis code is public domain and you should feel free to use it. I am paying for the hosting right now, so please be courteous :)\n\n```\nimport os\nimport time\nimport zipfile\n\nimport requests\nimport torch\nimport torch.utils.data as data\n\n\ndef recursive_listdir(path: str, suffix: str) -> list:\n return [os.path.join(dp, f)\n for dp, dn, fn in os.walk(os.path.expanduser(path))\n for f in fn\n if f.endswith(suffix)]\n\n\nclass GraspAndLiftEEGDataset(data.Dataset):\n \"\"\" 32-channel, 500Hz EEG dataset of subjects performing a various\n grasp-and-lift motor tasks, with per-sample class labels.\n\n There are 12 subjects in total, 10 series of trials for each subject,\n and approximately 30 trials within each series. The number of trials\n varies for each series. The training set contains the first 8 series\n for each subject. The test set contains the 9th and 10th series.\n\n Videos of trials being performed:\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM69_ESM.avi\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM70_ESM.avi\n (Note: these videos are also available in the S3 bucket)\n\n Args:\n root: Path to directory containing train/ and test/ folders\n\n train: If true, use the train/ directory and load class labels.\n If false, load the test/ directory (which lacks labels)\n\n download: If true, download the data if it is not present locally\n\n num_samples: The number of samples in the returned examples.\n If None, the the dataset yields the full length trials.\n\n last_label_only: If true, return only the last sample's labels.\n Must be used with num_samples.\n\n Examples:\n >>> dataset = GraspAndLiftEEGDataset('/data', train=True, download=True, num_samples=1024)\n Downloading from https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip\n Downloaded in 283 seconds\n Extracting /data/grasp-and-lift-eeg-detection.zip to /data\n Unzipped in 36 seconds\n >>> len(dataset)\n 17887546\n >>> data, label = dataset[0]\n >>> data.shape\n torch.Size([32, 1024])\n >>> label.shape\n torch.Size([6, 1024])\n\n Labels:\n 1. HandStart\n 2. FirstDigitTouch\n 3. BothStartLoadPhase\n 4. LiftOff\n 5. Replace\n 6. BothReleased\n\n Reference:\n Luciw, M., Jarocka, E. & Edin, B. Multi-channel EEG recordings during 3,936\n grasp and lift trials with varying weight and friction. Sci Data 1, 140047\n (2014). https://doi.org/10.1038/sdata.2014.47\n\n https://www.kaggle.com/c/grasp-and-lift-eeg-detection\n\n License:\n Original data has been made available under the terms of Attribution 4.0\n International Creative Commons License (http://creativecommons.org/licenses/by/4.0/).\n \"\"\"\n\n GRASPLIFT_DATA_HEADER = 'id,Fp1,Fp2,F7,F3,Fz,F4,F8,FC5,FC1,FC2,FC6,T7,C3,Cz,C4,T8,TP9,CP5,CP1,CP2,CP6,' + \\\n 'TP10,P7,P3,Pz,P4,P8,PO9,O1,Oz,O2,PO10\\n'\n\n GRASPLIFT_EVENTS_HEADER = 'id,HandStart,FirstDigitTouch,BothStartLoadPhase,LiftOff,Replace,BothReleased\\n'\n\n ZIP_URL = 'https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip'\n\n ZIP_SIZE_BYTES = 980887394\n\n def __init__(self,\n root: str,\n train: bool = True,\n download: bool = True,\n num_samples: int = None,\n last_label_only: bool = False):\n super(GraspAndLiftEEGDataset, self).__init__()\n if num_samples is None and last_label_only:\n raise ValueError('last_label_only cannot be used without setting num_samples')\n self.num_samples = num_samples\n self.last_label_only = last_label_only\n data_dir = os.path.join(root, 'train' if train else 'test')\n if not os.path.exists(data_dir):\n if not download:\n raise ValueError(f'{data_dir} does not exist')\n if not os.path.exists(root):\n os.makedirs(root)\n self.download(root)\n csv_suffix = '.csv'\n bin_suffix = '.csv.bin'\n csv_files = recursive_listdir(data_dir, csv_suffix)\n bin_files = recursive_listdir(data_dir, bin_suffix)\n should_compile = len(bin_files) < len(csv_files)\n if should_compile:\n print(f'Number of .csv.bin files ({len(bin_files)}) '\n f'is less than the number of .csv ({len(csv_files)}).'\n ' Compiling binary representation...')\n self.load_from_csv(csv_files)\n else:\n self.load_from_bin(bin_files)\n\n def load_from_csv(self, csv_files):\n self.X, self.Y = self.compile_bin(csv_files)\n if self.num_samples is not None:\n # Divide each example up into windows\n self.total_examples = 0\n for x in self.X:\n self.total_examples += x.shape[1] - self.num_samples + 1\n\n def load_from_bin(self, bin_files: list):\n examples = {}\n self.total_examples = 0\n for file in bin_files:\n is_data = file.endswith('_data.csv.bin')\n series = file[:-len('_data.csv.bin')\n if is_data else -len('_events.csv.bin')]\n samples = torch.load(file)\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n if is_data and self.num_samples is not None:\n self.total_examples += samples.shape[1] - self.num_samples + 1\n self.X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n self.X.append(x)\n if y is not None:\n Y.append(y)\n self.Y = Y if len(Y) > 0 else None\n\n def download(self, root: str):\n zip_path = os.path.join(root, 'grasp-and-lift-eeg-detection.zip')\n if not os.path.exists(zip_path) or os.path.getsize(zip_path) != self.ZIP_SIZE_BYTES:\n print(f'Downloading from {self.ZIP_URL}')\n start = time.time()\n r = requests.get(self.ZIP_URL)\n if r.status_code != 200:\n raise ValueError(\n f'Expected status code 200, got {r.status_code}')\n with open(zip_path, 'wb') as f:\n f.write(r.content)\n delta = time.time() - start\n print(f'Downloaded in {int(delta)} seconds')\n print(f'Extracting {zip_path} to {root}')\n start = time.time()\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(root)\n delta = time.time() - start\n print(f'Unzipped in {int(delta)} seconds')\n os.remove(zip_path)\n\n def compile_bin(self, csv_files: list):\n examples = {}\n for i, file in enumerate(csv_files):\n is_data = file.endswith('_data.csv')\n samples = []\n with open(file, 'r') as f:\n hdr = f.readline()\n expected_hdr = self.GRASPLIFT_DATA_HEADER if is_data else self.GRASPLIFT_EVENTS_HEADER\n if hdr != expected_hdr:\n raise ValueError('bad header')\n for line in f:\n channels = line.strip().split(',')[1:]\n if is_data:\n # Data is converted to float eventually anyway\n channels = [float(x) for x in channels]\n else:\n # Labels are integer format\n channels = [int(x) for x in channels]\n channels = torch.Tensor(channels).unsqueeze(1)\n samples.append(channels)\n samples = torch.cat(samples, dim=1)\n series = file[:-len('_data.csv')\n if is_data else -len('_events.csv')]\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n print(f'Processed {i+1}/{len(csv_files)} {file}')\n X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n torch.save(samples, series + '_data.csv.bin')\n X.append(x)\n if y is not None:\n torch.save(samples, series + '_events.csv.bin')\n Y.append(y)\n return X, Y if len(Y) > 0 else None\n\n def __getitem__(self, index):\n if self.num_samples is None:\n # Return the entire example (e.g. reinforcement learning)\n return (self.X[index], self.Y[index] if self.Y is not None else [])\n # Find the example and offset for the index\n ofs = 0\n for i, x in enumerate(self.X):\n num_examples = x.shape[1] - self.num_samples + 1\n if index >= ofs + num_examples:\n ofs += num_examples\n continue\n j = index - ofs\n x = x[:, j:j + self.num_samples]\n if self.Y is not None:\n if self.last_label_only:\n # Only return the label for the last sample\n y = self.Y[i][:, j + self.num_samples - 1]\n else:\n # Return labels for all samples\n y = self.Y[i][:, j:j + self.num_samples]\n else:\n y = []\n return x, y\n raise ValueError(f'unable to seek {index}')\n\n def __len__(self):\n if self.num_samples is None:\n # No windowing - each example is full length\n return len(self.X)\n # Use precalculated dataset length\n return self.total_examples\n```\n\nThe latest version (if updated) is available at: https://github.com/thavlik/machine-learning-portfolio/blob/master/src/dataset/grasp_and_lift_eeg.py\n\nBig thanks to WAY Consortium for making this dataset freely available!" } ], "comments": [], "raw_markdown_by_id": { "1113357": "I'm currently trying to contribute this Dataset class to pytorch-lightning-bolts. It converts the csv files into binary, resulting in significant performance gains.\n\nThis code is public domain and you should feel free to use it. I am paying for the hosting right now, so please be courteous :)\n\n```\nimport os\nimport time\nimport zipfile\n\nimport requests\nimport torch\nimport torch.utils.data as data\n\n\ndef recursive_listdir(path: str, suffix: str) -> list:\n return [os.path.join(dp, f)\n for dp, dn, fn in os.walk(os.path.expanduser(path))\n for f in fn\n if f.endswith(suffix)]\n\n\nclass GraspAndLiftEEGDataset(data.Dataset):\n \"\"\" 32-channel, 500Hz EEG dataset of subjects performing a various\n grasp-and-lift motor tasks, with per-sample class labels.\n\n There are 12 subjects in total, 10 series of trials for each subject,\n and approximately 30 trials within each series. The number of trials\n varies for each series. The training set contains the first 8 series\n for each subject. The test set contains the 9th and 10th series.\n\n Videos of trials being performed:\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM69_ESM.avi\n https://static-content.springer.com/esm/art%3A10.1038%2Fsdata.2014.47/MediaObjects/41597_2014_BFsdata201447_MOESM70_ESM.avi\n (Note: these videos are also available in the S3 bucket)\n\n Args:\n root: Path to directory containing train/ and test/ folders\n\n train: If true, use the train/ directory and load class labels.\n If false, load the test/ directory (which lacks labels)\n\n download: If true, download the data if it is not present locally\n\n num_samples: The number of samples in the returned examples.\n If None, the the dataset yields the full length trials.\n\n last_label_only: If true, return only the last sample's labels.\n Must be used with num_samples.\n\n Examples:\n >>> dataset = GraspAndLiftEEGDataset('/data', train=True, download=True, num_samples=1024)\n Downloading from https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip\n Downloaded in 283 seconds\n Extracting /data/grasp-and-lift-eeg-detection.zip to /data\n Unzipped in 36 seconds\n >>> len(dataset)\n 17887546\n >>> data, label = dataset[0]\n >>> data.shape\n torch.Size([32, 1024])\n >>> label.shape\n torch.Size([6, 1024])\n\n Labels:\n 1. HandStart\n 2. FirstDigitTouch\n 3. BothStartLoadPhase\n 4. LiftOff\n 5. Replace\n 6. BothReleased\n\n Reference:\n Luciw, M., Jarocka, E. & Edin, B. Multi-channel EEG recordings during 3,936\n grasp and lift trials with varying weight and friction. Sci Data 1, 140047\n (2014). https://doi.org/10.1038/sdata.2014.47\n\n https://www.kaggle.com/c/grasp-and-lift-eeg-detection\n\n License:\n Original data has been made available under the terms of Attribution 4.0\n International Creative Commons License (http://creativecommons.org/licenses/by/4.0/).\n \"\"\"\n\n GRASPLIFT_DATA_HEADER = 'id,Fp1,Fp2,F7,F3,Fz,F4,F8,FC5,FC1,FC2,FC6,T7,C3,Cz,C4,T8,TP9,CP5,CP1,CP2,CP6,' + \\\n 'TP10,P7,P3,Pz,P4,P8,PO9,O1,Oz,O2,PO10\\n'\n\n GRASPLIFT_EVENTS_HEADER = 'id,HandStart,FirstDigitTouch,BothStartLoadPhase,LiftOff,Replace,BothReleased\\n'\n\n ZIP_URL = 'https://grasplifteeg.nyc3.digitaloceanspaces.com/grasp-and-lift-eeg-detection.zip'\n\n ZIP_SIZE_BYTES = 980887394\n\n def __init__(self,\n root: str,\n train: bool = True,\n download: bool = True,\n num_samples: int = None,\n last_label_only: bool = False):\n super(GraspAndLiftEEGDataset, self).__init__()\n if num_samples is None and last_label_only:\n raise ValueError('last_label_only cannot be used without setting num_samples')\n self.num_samples = num_samples\n self.last_label_only = last_label_only\n data_dir = os.path.join(root, 'train' if train else 'test')\n if not os.path.exists(data_dir):\n if not download:\n raise ValueError(f'{data_dir} does not exist')\n if not os.path.exists(root):\n os.makedirs(root)\n self.download(root)\n csv_suffix = '.csv'\n bin_suffix = '.csv.bin'\n csv_files = recursive_listdir(data_dir, csv_suffix)\n bin_files = recursive_listdir(data_dir, bin_suffix)\n should_compile = len(bin_files) < len(csv_files)\n if should_compile:\n print(f'Number of .csv.bin files ({len(bin_files)}) '\n f'is less than the number of .csv ({len(csv_files)}).'\n ' Compiling binary representation...')\n self.load_from_csv(csv_files)\n else:\n self.load_from_bin(bin_files)\n\n def load_from_csv(self, csv_files):\n self.X, self.Y = self.compile_bin(csv_files)\n if self.num_samples is not None:\n # Divide each example up into windows\n self.total_examples = 0\n for x in self.X:\n self.total_examples += x.shape[1] - self.num_samples + 1\n\n def load_from_bin(self, bin_files: list):\n examples = {}\n self.total_examples = 0\n for file in bin_files:\n is_data = file.endswith('_data.csv.bin')\n series = file[:-len('_data.csv.bin')\n if is_data else -len('_events.csv.bin')]\n samples = torch.load(file)\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n if is_data and self.num_samples is not None:\n self.total_examples += samples.shape[1] - self.num_samples + 1\n self.X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n self.X.append(x)\n if y is not None:\n Y.append(y)\n self.Y = Y if len(Y) > 0 else None\n\n def download(self, root: str):\n zip_path = os.path.join(root, 'grasp-and-lift-eeg-detection.zip')\n if not os.path.exists(zip_path) or os.path.getsize(zip_path) != self.ZIP_SIZE_BYTES:\n print(f'Downloading from {self.ZIP_URL}')\n start = time.time()\n r = requests.get(self.ZIP_URL)\n if r.status_code != 200:\n raise ValueError(\n f'Expected status code 200, got {r.status_code}')\n with open(zip_path, 'wb') as f:\n f.write(r.content)\n delta = time.time() - start\n print(f'Downloaded in {int(delta)} seconds')\n print(f'Extracting {zip_path} to {root}')\n start = time.time()\n with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n zip_ref.extractall(root)\n delta = time.time() - start\n print(f'Unzipped in {int(delta)} seconds')\n os.remove(zip_path)\n\n def compile_bin(self, csv_files: list):\n examples = {}\n for i, file in enumerate(csv_files):\n is_data = file.endswith('_data.csv')\n samples = []\n with open(file, 'r') as f:\n hdr = f.readline()\n expected_hdr = self.GRASPLIFT_DATA_HEADER if is_data else self.GRASPLIFT_EVENTS_HEADER\n if hdr != expected_hdr:\n raise ValueError('bad header')\n for line in f:\n channels = line.strip().split(',')[1:]\n if is_data:\n # Data is converted to float eventually anyway\n channels = [float(x) for x in channels]\n else:\n # Labels are integer format\n channels = [int(x) for x in channels]\n channels = torch.Tensor(channels).unsqueeze(1)\n samples.append(channels)\n samples = torch.cat(samples, dim=1)\n series = file[:-len('_data.csv')\n if is_data else -len('_events.csv')]\n item = examples.get(series, [None, None])\n item[0 if is_data else 1] = samples\n examples[series] = item\n print(f'Processed {i+1}/{len(csv_files)} {file}')\n X = []\n Y = []\n for series in sorted(examples):\n x, y = examples[series]\n torch.save(samples, series + '_data.csv.bin')\n X.append(x)\n if y is not None:\n torch.save(samples, series + '_events.csv.bin')\n Y.append(y)\n return X, Y if len(Y) > 0 else None\n\n def __getitem__(self, index):\n if self.num_samples is None:\n # Return the entire example (e.g. reinforcement learning)\n return (self.X[index], self.Y[index] if self.Y is not None else [])\n # Find the example and offset for the index\n ofs = 0\n for i, x in enumerate(self.X):\n num_examples = x.shape[1] - self.num_samples + 1\n if index >= ofs + num_examples:\n ofs += num_examples\n continue\n j = index - ofs\n x = x[:, j:j + self.num_samples]\n if self.Y is not None:\n if self.last_label_only:\n # Only return the label for the last sample\n y = self.Y[i][:, j + self.num_samples - 1]\n else:\n # Return labels for all samples\n y = self.Y[i][:, j:j + self.num_samples]\n else:\n y = []\n return x, y\n raise ValueError(f'unable to seek {index}')\n\n def __len__(self):\n if self.num_samples is None:\n # No windowing - each example is full length\n return len(self.X)\n # Use precalculated dataset length\n return self.total_examples\n```\n\nThe latest version (if updated) is available at: https://github.com/thavlik/machine-learning-portfolio/blob/master/src/dataset/grasp_and_lift_eeg.py\n\nBig thanks to WAY Consortium for making this dataset freely available!" } }