{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-29T18:54:48.226683Z","iopub.execute_input":"2023-03-29T18:54:48.227785Z","iopub.status.idle":"2023-03-29T18:54:48.237318Z","shell.execute_reply.started":"2023-03-29T18:54:48.227736Z","shell.execute_reply":"2023-03-29T18:54:48.235890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import SGD, Adam\nfrom torch import nn\nfrom torchmetrics.classification.accuracy import MulticlassAccuracy\n\nimport pylab","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.240261Z","iopub.execute_input":"2023-03-29T18:54:48.240747Z","iopub.status.idle":"2023-03-29T18:54:48.246718Z","shell.execute_reply.started":"2023-03-29T18:54:48.240709Z","shell.execute_reply":"2023-03-29T18:54:48.245483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First let's set the random seed, so that we always get the same results when running the code.\nRemember, this isn't a hyperparameter, we aren't looking for the lotto ticket number.","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(0)","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.248757Z","iopub.execute_input":"2023-03-29T18:54:48.249270Z","iopub.status.idle":"2023-03-29T18:54:48.257925Z","shell.execute_reply.started":"2023-03-29T18:54:48.249234Z","shell.execute_reply":"2023-03-29T18:54:48.256898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to define a torch.utils.data.Dataset type that reads our raw data and returns a (sample, label) tuple for each sample.","metadata":{}},{"cell_type":"code","source":"class WaveformDataset(Dataset):\n    def __init__(self, npz_file, test=False):\n        df = np.load(npz_file)\n        self.foregrounds = df['foreground']\n        self.n_samples = self.foregrounds.shape[0]\n        print(f'Total number of samples: {self.n_samples}')\n        self.test = test\n        if not self.test:\n            # we don't have labels available for the test set\n            self.labels = df['label'][:self.n_samples]\n\n    def __len__(self):\n        return self.n_samples\n\n    def __getitem__(self, item):\n        fg = np.asarray(self.foregrounds[item, :])\n        if self.test:\n            return torch.from_numpy(fg)\n\n        label = np.asarray(self.labels[item])\n        return torch.from_numpy(fg), label\n\ntrain_dataset = WaveformDataset('/kaggle/input/g2net-hackathon/classification_training_data.npz')\nval_dataset = WaveformDataset('/kaggle/input/g2net-hackathon/classification_validation_data.npz')","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.260587Z","iopub.execute_input":"2023-03-29T18:54:48.260979Z","iopub.status.idle":"2023-03-29T18:54:48.503820Z","shell.execute_reply.started":"2023-03-29T18:54:48.260943Z","shell.execute_reply":"2023-03-29T18:54:48.502518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time to create our network architecture. \nCan you think of any reason not to use only nn.Linear layers? What about only nn.Conv1d layers?\nCan the nn.MaxPool1d layers be replaced with other types of layers that equivalently reduce the temporal dimension?\nWhat about the non-linearities?","metadata":{}},{"cell_type":"code","source":"network = nn.Sequential(  # Shapes\n    nn.BatchNorm1d(1),  # 1x2048\n    nn.Conv1d(1, 4, 64),  # 4x1985\n    nn.ELU(),  # 4x1985\n    nn.Conv1d(4, 4, 32),  # 4x1954\n    nn.MaxPool1d(4),  # 4x 489\n    nn.ELU(),  # 4x 489\n    nn.Conv1d(4, 8, 32),  # 8x 458\n    nn.ELU(),  # 8x 458\n    nn.Conv1d(8, 8, 16),  # 8x 443\n    nn.MaxPool1d(3),  # 8x 147\n    nn.ELU(),  # 8x 147\n    nn.Conv1d(8, 16, 16),  # 16x 132\n    nn.ELU(),  # 16x 132\n    nn.Conv1d(16, 16, 16),  # 16x 117\n    nn.MaxPool1d(4),  # 16x  29\n    nn.ELU(),  # 16x  29\n    nn.Flatten(),  # 464\n    nn.Linear(464, 32),  # 32\n    nn.Dropout(p=0.5),  # 32\n    nn.ELU(),  # 32\n    nn.Linear(32, 16),  # 16\n    nn.Dropout(p=0.5),  # 16\n    nn.ELU(),  # 16\n    nn.Linear(16, 3),  # 2\n    nn.Softmax(dim=1)  # 2\n)\nprint(network)\n# HINT: https://pytorch.org/docs/stable/nn.init.html\n# https://stackoverflow.com/a/49433937","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.506247Z","iopub.execute_input":"2023-03-29T18:54:48.506653Z","iopub.status.idle":"2023-03-29T18:54:48.521686Z","shell.execute_reply.started":"2023-03-29T18:54:48.506612Z","shell.execute_reply":"2023-03-29T18:54:48.520315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's define some hyperparameters and load our datasets.\nWhat happens when you try different optimizers? What about different learning rates or batch sizes?\nRemember, a tried and true strategy regarding the learning rate is to reduce it as training progresses.\nAre there other loss functions that can be used instead of nn.CrossEntropyLoss?","metadata":{}},{"cell_type":"code","source":"device = 'cuda:0'\nnetwork = network.to(device)\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=2, pin_memory=device)\nval_dataloader = DataLoader(val_dataset, batch_size=128, num_workers=2, pin_memory=device)\nn_epochs = 10\nlr = 5e-5\noptimizer = Adam(network.parameters(), lr=lr)\n# HINT: https://pytorch.org/docs/stable/generated/torch.optim.lr_scheduler.StepLR.html\nloss = nn.CrossEntropyLoss()\n\ntrain_losses = np.zeros((n_epochs,))\ntrain_accs = np.zeros((n_epochs,))\nval_losses = np.zeros((n_epochs,))\nval_accs = np.zeros((n_epochs,))\n\nacc_metric = MulticlassAccuracy(num_classes=3, average='none').to(device)","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.523738Z","iopub.execute_input":"2023-03-29T18:54:48.524186Z","iopub.status.idle":"2023-03-29T18:54:48.537540Z","shell.execute_reply.started":"2023-03-29T18:54:48.524150Z","shell.execute_reply":"2023-03-29T18:54:48.536473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ready for our training loop.\nRemember, in mini-btach gradient descent, we say one epoch has passed when the network has seen the training set in its entirety.\nAn iteration is when we pass a single mini-batch through the network, compute the loss and gradients, and update the weights.","metadata":{}},{"cell_type":"code","source":"for ep in range(n_epochs):\n    network.train()\n    training_loss = 0.\n    training_correct = 0\n    training_total = 0\n    for i, (x, y) in enumerate(train_dataloader):\n        # unsqueeze is needed here to add a channel dimension so we can use convolutional layers\n        x = x.unsqueeze_(1).to(dtype=torch.float32).to(device)\n        y = y.to(dtype=torch.long).to(device)\n\n        optimizer.zero_grad()\n\n        p = network(x)\n        l = loss(p, y)\n\n        l.backward()\n        # HINT: https://pytorch.org/docs/stable/generated/torch.nn.utils.clip_grad_norm_.html\n        optimizer.step()\n        \n        training_loss += l.item()\n        # we need to convert the one-hot predictions into (0, 1, 2) labels to measure accuracy\n        _, p_c = p.max(1)\n        training_correct += p_c.eq(y).sum().item()\n        training_total += x.size(0)\n\n    train_losses[ep] = training_loss / (i + 1)\n    train_accs[ep] = 100 * training_correct/training_total\n    print(f'Epoch {ep} | Training   | Loss {train_losses[ep]:3.2f} |'\n          f' Acc {train_accs[ep]:5.2f}% ')\n\n    network.eval()  # this line sets the network to evaluation mode (e.g., do not compute gradients, or mean and std for batch normalization layers)\n    validation_loss = 0.\n    validation_correct = 0.\n    validation_total = 0.\n    val_preds = []\n    val_gt = []\n    with torch.no_grad():\n        for i, (x, y) in enumerate(val_dataloader):\n            x = x.unsqueeze_(1).to(dtype=torch.float32).to(device)\n            y = y.to(dtype=torch.long).to(device)\n            p = network(x)\n            l = loss(p, y)\n            # no backward pass for validation set\n            validation_loss += l.item()\n            _, p_c = p.max(1)\n            val_preds.extend(p_c)\n            val_gt.extend(y)\n            validation_correct += p_c.eq(y).sum().item()\n            validation_total += x.size(0)\n        val_losses[ep] = validation_loss / (i + 1)\n        val_accs[ep] = 100 * validation_correct / validation_total\n        val_acc_per_class = acc_metric(torch.stack(val_preds), torch.stack(val_gt))\n        print(f'Epoch {ep} | Validation | Loss {val_losses[ep]:3.2f} |'\n              f' Acc {val_accs[ep]:5.2f}% (0:{val_acc_per_class[0]:5.2f} 1:{val_acc_per_class[1]:5.2f} 2:{val_acc_per_class[2]:5.2f})')","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:54:48.540732Z","iopub.execute_input":"2023-03-29T18:54:48.541075Z","iopub.status.idle":"2023-03-29T18:55:23.348188Z","shell.execute_reply.started":"2023-03-29T18:54:48.541041Z","shell.execute_reply":"2023-03-29T18:55:23.346641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's visualize the training progress in terms of train/val loss/accuracy.\nWhat do these curves tell us about which hyperparameters we should change?\nRemember, change only one hyperparameter at a time, but write down all of them.","metadata":{}},{"cell_type":"code","source":"fig, axs = pylab.subplots(1, 2, sharex=True)\naxs[0].plot(train_losses, label='train', color='xkcd:blue')\naxs[0].plot(val_losses, label='val', color='xkcd:red')\naxs[0].set_title('Loss')\naxs[0].set_xlabel('Epochs')\n\naxs[1].plot(train_accs, label='train', color='xkcd:blue')\naxs[1].plot(val_accs, label='val', color='xkcd:red')\naxs[1].set_title('Accuracy')\naxs[1].set_xlabel('Epochs')","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:55:23.350444Z","iopub.execute_input":"2023-03-29T18:55:23.351166Z","iopub.status.idle":"2023-03-29T18:55:23.687183Z","shell.execute_reply.started":"2023-03-29T18:55:23.351105Z","shell.execute_reply":"2023-03-29T18:55:23.686204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Time to run our network on the test data. We only have the input timeseries available (not the labels), so we need a slightly modified loop.","metadata":{}},{"cell_type":"code","source":"print('Running network on test data...')\ntest_dataset = WaveformDataset('/kaggle/input/g2net-hackathon/classification_test_data.npz', test=True)\n# IMPORTANT: do NOT shuffle test set\ntest_dataloader = DataLoader(test_dataset, batch_size=128, num_workers=8, pin_memory=device, shuffle=False)\nnetwork.eval()\ntest_preds = []\ntest_gt = []\nwith torch.no_grad():\n    for i, x in enumerate(test_dataloader):\n        x = x.unsqueeze_(1).to(dtype=torch.float32).to(device)\n        p = network(x)\n        \n        _, p_c = p.max(1)\n        test_preds.extend(p_c.cpu().numpy())\n# write result to csv\ndf = pd.DataFrame({'Category': np.asarray(test_preds)})\ndf.index.name = 'Id'\ndf.to_csv('classification_test_predictions.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-29T18:55:23.688606Z","iopub.execute_input":"2023-03-29T18:55:23.689067Z","iopub.status.idle":"2023-03-29T18:55:24.725119Z","shell.execute_reply.started":"2023-03-29T18:55:23.689029Z","shell.execute_reply":"2023-03-29T18:55:24.723887Z"},"trusted":true},"execution_count":null,"outputs":[]}]}