{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torchsummary torchview","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:36:04.420246Z","iopub.execute_input":"2023-10-21T14:36:04.421154Z","iopub.status.idle":"2023-10-21T14:36:16.659014Z","shell.execute_reply.started":"2023-10-21T14:36:04.421116Z","shell.execute_reply":"2023-10-21T14:36:16.657817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport csv\nimport cv2 as cv\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport seaborn as sns\nimport shutil\nimport time\n\nfrom torchvision import transforms\nfrom torchvision.transforms import v2\nfrom sklearn.metrics import accuracy_score\nfrom matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch import nn\nfrom torchsummary import summary\nfrom torchview import draw_graph","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-21T15:04:30.790634Z","iopub.execute_input":"2023-10-21T15:04:30.791551Z","iopub.status.idle":"2023-10-21T15:04:30.798080Z","shell.execute_reply.started":"2023-10-21T15:04:30.791517Z","shell.execute_reply":"2023-10-21T15:04:30.797018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils","metadata":{}},{"cell_type":"code","source":"def read_data(filename):\n    with open(filename, \"r\") as file:\n        reader = csv.reader(file)\n        reader.__next__()\n\n        for row in reader:\n            yield np.array(row, dtype=float)\n\n\ndef get_label_and_img(array):\n    return array[1:], array[0]\n\n\ndef convert_to_img(array, filename):\n    img = array.reshape((28, 28))\n    cv.imwrite(filename, img)\n\n\ndef do_preprocess(filename, dst_dir):\n    Y = []\n\n    for i, arr in enumerate(read_data(filename)):\n        img, label = get_label_and_img(arr)\n        file = f'img_{i}.jpg'\n\n        Y.append([file, label])\n        dst = os.path.join(dst_dir, file)\n        convert_to_img(img, dst)\n\n    df = pd.DataFrame(Y, columns=['file', 'label'])\n    df.to_csv('label.csv', index=False)","metadata":{"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2023-10-21T14:36:32.802863Z","iopub.execute_input":"2023-10-21T14:36:32.803459Z","iopub.status.idle":"2023-10-21T14:36:32.812017Z","shell.execute_reply.started":"2023-10-21T14:36:32.803426Z","shell.execute_reply":"2023-10-21T14:36:32.811003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not os.path.isdir('./train'):\n    os.mkdir('./train')\n    \nif not os.path.isdir('./test'):\n    os.mkdir('./test')\n\ndo_preprocess('../input/digit-recognizer/train.csv', './train')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:36:35.420913Z","iopub.execute_input":"2023-10-21T14:36:35.421294Z","iopub.status.idle":"2023-10-21T14:36:47.430320Z","shell.execute_reply.started":"2023-10-21T14:36:35.421265Z","shell.execute_reply":"2023-10-21T14:36:47.429395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img_files = os.listdir('./train/')\ntest_sample = np.random.choice(img_files, size=256, replace=False)\n\nfor file in tqdm(test_sample):\n    curr = os.path.join('./train', file)\n    new = os.path.join('./test', file)\n\n    shutil.move(curr, new)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:36:47.431898Z","iopub.execute_input":"2023-10-21T14:36:47.432237Z","iopub.status.idle":"2023-10-21T14:36:47.499724Z","shell.execute_reply.started":"2023-10-21T14:36:47.432206Z","shell.execute_reply":"2023-10-21T14:36:47.498872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv('./label.csv')\ntest_set = labels[labels['file'].isin(test_sample)]\nlabels.drop(test_set.index, axis=0).to_csv('./label.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:36:50.120884Z","iopub.execute_input":"2023-10-21T14:36:50.121233Z","iopub.status.idle":"2023-10-21T14:36:50.292937Z","shell.execute_reply.started":"2023-10-21T14:36:50.121206Z","shell.execute_reply":"2023-10-21T14:36:50.292164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def move_to(obj, device):\n    if isinstance(obj, list):\n        return [move_to(x, device) for x in obj]\n    \n    elif isinstance(obj, tuple):\n        return tuple(move_to(obj, device))\n    \n    elif isinstance(obj, set):\n        return set(move_to(obj, device))\n    \n    elif isinstance(obj, dict):\n        to_return = dict()\n        \n        for key, value in obj.items():\n            to_return[move_to(key, device)] = move_to(value, device)\n        \n        return to_return\n    \n    elif hasattr(obj, 'to'):\n        return obj.to(device)\n    \n    else:\n        return obj","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:36:55.571358Z","iopub.execute_input":"2023-10-21T14:36:55.571716Z","iopub.status.idle":"2023-10-21T14:36:55.579170Z","shell.execute_reply.started":"2023-10-21T14:36:55.571687Z","shell.execute_reply":"2023-10-21T14:36:55.578172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def run_epoch(model, optimizer, data_loader,\n              loss_func, device, results,\n              score_funcs, prefix=\"\", desc=None, keep=False, epoch=None):\n    running_loss = []\n    y_true = []\n    y_pred = []\n    start = time.time()\n\n    for inputs, labels in tqdm(data_loader, desc=desc, leave=keep):\n        # Move the batch to the device we are using.\n        inputs = move_to(inputs, device)\n        labels = move_to(labels, device)\n\n        y_hat, mu, sigma = model(inputs)  # this just computed f_Θ(x(i))\n        \n        # Compute loss.\n        reconstructed_loss = loss_func(y_hat, labels)\n        kl_div = -torch.sum(1 + torch.log(sigma.pow(2)) - mu.pow(2) - sigma.pow(2))\n        loss = reconstructed_loss + kl_div\n\n        if model.training:\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n\n        # Now we are just grabbing some information we would like to have\n        running_loss.append(loss.item())\n\n        if len(score_funcs) > 0 and isinstance(labels, torch.Tensor):\n            # moving labels & predictions back to CPU for computing / storing predictions\n            labels = labels.detach().cpu().numpy()\n            y_hat = y_hat.detach().cpu().numpy()\n            # add to predictions so far\n            y_true.extend(labels.tolist())\n            y_pred.extend(y_hat.tolist())\n\n    # end training epoch\n    end = time.time()\n\n    y_pred = np.asarray(y_pred)\n    if len(y_pred.shape) == 2 and y_pred.shape[1] > 1:  # We have a classification problem, convert to labels\n        y_pred = np.argmax(y_pred, axis=1)\n    # Else, we assume we are working on a regression problem\n\n    results[prefix + \" loss\"].append(np.mean(running_loss))\n    result_str = [f\"{prefix} loss: {np.mean(running_loss)}\"]\n\n    for name, score_func in score_funcs.items():\n        try:\n            score = score_func(y_true, y_pred)\n\n            results[prefix + \" \" + name].append(score)\n            result_str.append(f\"{prefix} {name}: {score}\")\n\n            if prefix == 'val' or prefix == 'test':\n                checkpointer(score, (epoch + 1), model, optimizer)\n\n        except:\n            results[prefix + \" \" + name].append(float(\"NaN\"))\n            result_str.append(f\"{prefix} {name}: {float('NaN')}\")\n\n    print(\" \".join(result_str))\n\n    return end - start  # time spent on epoch","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:39:17.582628Z","iopub.execute_input":"2023-10-21T14:39:17.583025Z","iopub.status.idle":"2023-10-21T14:39:17.596594Z","shell.execute_reply.started":"2023-10-21T14:39:17.582988Z","shell.execute_reply":"2023-10-21T14:39:17.595624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_loop(model, loss_func,\n               train_loader, test_loader=None, val_loader=None,\n               score_funcs=None, epochs=50, device=\"cpu\",\n               optimizer=None, lr_schedule=None, checkpoint_file=None, keep=True):\n    if score_funcs is None:\n        score_funcs = {}\n\n    to_track = [\"epoch\", \"total time\", \"train loss\", \"lr\"]\n\n    if val_loader is not None:\n        to_track.append(\"val loss\")\n\n    if test_loader is not None:\n        to_track.append(\"test loss\")\n\n    for eval_score in score_funcs:\n        to_track.append(\"train \" + eval_score)\n\n        if val_loader is not None:\n            to_track.append(\"val \" + eval_score)\n\n        if test_loader is not None:\n            to_track.append(\"test \" + eval_score)\n\n    total_train_time = 0  # How long have we spent in the training loop?\n    results = {}\n\n    # Initialize every item with an empty list\n    for item in to_track:\n        results[item] = []\n\n    if optimizer is None:\n        optimizer = torch.optim.AdamW(model.parameters(), lr=0.001)\n        del_opt = True\n\n    else:\n        del_opt = False\n\n    # Place the model on the correct computed resource (CPU or GPU)\n    model.to(device)\n\n    for epoch in tqdm(range(epochs), desc=\"Epoch\", leave=keep):\n        model = model.train()  # Put our model in training mode\n\n        run_epoch(model, optimizer, train_loader, loss_func, device, results, score_funcs,\n                  prefix=\"train\", desc=\"Training\")\n\n        results[\"total time\"].append(total_train_time)\n        results[\"epoch\"].append(epoch)\n        results[\"lr\"].append(optimizer.param_groups[0]['lr'])\n\n        if val_loader is not None:\n            model = model.eval()\n\n            with torch.no_grad():\n                run_epoch(model, optimizer, val_loader, loss_func, device, results, score_funcs, prefix=\"val\",\n                          desc=\"Validation\", epoch=epoch)\n\n        if test_loader is not None:\n            model = model.eval()\n\n            with torch.no_grad():\n                run_epoch(model, optimizer, test_loader, loss_func, device, results, score_funcs, prefix=\"test\",\n                          desc=\"Testing\", epoch=epoch)\n\n        if lr_schedule is not None:\n            if isinstance(lr_schedule, torch.optim.lr_scheduler.ReduceLROnPlateau):\n                lr_schedule.step(results[\"val loss\"][-1])\n\n            else:\n                lr_schedule.step()\n\n        if checkpoint_file is not None:\n            torch.save({\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict(),\n                'results': results\n            }, checkpoint_file)\n\n    if del_opt:\n        del optimizer\n\n    return pd.DataFrame.from_dict(results)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:39:22.074202Z","iopub.execute_input":"2023-10-21T14:39:22.074529Z","iopub.status.idle":"2023-10-21T14:39:22.089502Z","shell.execute_reply.started":"2023-10-21T14:39:22.074504Z","shell.execute_reply":"2023-10-21T14:39:22.088516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load data ","metadata":{}},{"cell_type":"code","source":"class AutoEncodeDataset(Dataset):\n    \"\"\"loading input as output for x,x tuples in training data\"\"\"\n\n    def __init__(self, annotation_file, img_dir, transform=None):\n        self.img_labels = pd.read_csv(annotation_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.resize = v2.Compose([v2.Resize(size=24),\n                                  transforms.ToTensor()])\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_labels.loc[idx, 'file'])\n        image = torchvision.io.read_image(img_path).to(torch.float32) / 255\n\n        if self.transform is not None:\n            image = transforms.ToPILImage()(image)\n            image = self.transform(image)\n            \n        label = transforms.ToPILImage()(image)\n        label = self.resize(label)\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:39:43.741307Z","iopub.execute_input":"2023-10-21T14:39:43.742212Z","iopub.status.idle":"2023-10-21T14:39:43.750394Z","shell.execute_reply.started":"2023-10-21T14:39:43.742178Z","shell.execute_reply":"2023-10-21T14:39:43.749403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_transform = v2.Compose([v2.RandomAffine(degrees=5, translate=(0.05, 0.05), scale=(0.98, 1.02)),\n                              v2.RandomPerspective(distortion_scale=0.5, p=0.5),\n                              transforms.ToTensor()])","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:40:10.611091Z","iopub.execute_input":"2023-10-21T14:40:10.611477Z","iopub.status.idle":"2023-10-21T14:40:10.617004Z","shell.execute_reply.started":"2023-10-21T14:40:10.611448Z","shell.execute_reply":"2023-10-21T14:40:10.615921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data = AutoEncodeDataset('./label.csv', './train/', transform=train_transform)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:40:14.981467Z","iopub.execute_input":"2023-10-21T14:40:14.981795Z","iopub.status.idle":"2023-10-21T14:40:15.010418Z","shell.execute_reply.started":"2023-10-21T14:40:14.981770Z","shell.execute_reply":"2023-10-21T14:40:15.009423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = int(len(training_data) * 0.8)\ntest_size = len(training_data) - train_size\n\ntrain_dataset, test_dataset = random_split(training_data, (train_size, test_size))\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=256, shuffle=True)\ntest_dataloader = DataLoader(test_dataset, batch_size=256, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:40:29.660279Z","iopub.execute_input":"2023-10-21T14:40:29.661113Z","iopub.status.idle":"2023-10-21T14:40:29.691038Z","shell.execute_reply.started":"2023-10-21T14:40:29.661078Z","shell.execute_reply":"2023-10-21T14:40:29.690215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model building","metadata":{}},{"cell_type":"code","source":"if torch.cuda.is_available():\n    device = torch.device('cuda')\n    \nelse:\n    device = torch.device('cpu')","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:40:32.101840Z","iopub.execute_input":"2023-10-21T14:40:32.102655Z","iopub.status.idle":"2023-10-21T14:40:32.136965Z","shell.execute_reply.started":"2023-10-21T14:40:32.102621Z","shell.execute_reply":"2023-10-21T14:40:32.135999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:40:34.450699Z","iopub.execute_input":"2023-10-21T14:40:34.451067Z","iopub.status.idle":"2023-10-21T14:40:34.457810Z","shell.execute_reply.started":"2023-10-21T14:40:34.451036Z","shell.execute_reply":"2023-10-21T14:40:34.456924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Variational Autoencoder model","metadata":{}},{"cell_type":"code","source":"class Encode(nn.Module):\n\n    def __init__(self, channels, filters, embeddings, kernel):\n        super(Encode, self).__init__()\n\n        self.relu = nn.ReLU()\n        self.max_pool = nn.MaxPool2d(kernel_size=2)\n\n        self.conv1 = nn.Conv2d(channels, filters, kernel_size=kernel, padding=1)\n        self.bn1 = nn.BatchNorm2d(filters)\n        self.conv2 = nn.Conv2d(filters, filters // 2, kernel_size=kernel, padding=1)\n        self.bn2 = nn.BatchNorm2d(filters // 2)\n        self.conv3 = nn.Conv2d(filters // 2, filters // 4, kernel_size=kernel, padding=1)\n        self.bn3 = nn.BatchNorm2d(filters // 4)\n\n        self.mu = nn.Conv2d(filters // 4, embeddings, kernel_size=kernel, padding=1)\n        self.sigma = nn.Conv2d(filters // 4, embeddings, kernel_size=kernel, padding=1)\n\n    def forward(self, x):\n        x = self.bn1(self.relu(self.conv1(x)))\n        x = self.max_pool(x)\n        x = self.bn2(self.relu(self.conv2(x)))\n        x = self.max_pool(x)\n        x = self.bn3(self.relu(self.conv3(x)))\n        x = self.max_pool(x)\n\n        return self.mu(x), self.sigma(x)\n    \n    \nclass Decode(nn.Module):\n\n    def __init__(self, channels, filters, embeddings, kernel):\n        super(Decode, self).__init__()\n\n        self.relu = nn.ReLU()\n        self.deconv1 = nn.ConvTranspose2d(embeddings, filters // 4, kernel_size=kernel, stride=2, dilation=1,\n                                          output_padding=1, padding=1)\n        self.bn1 = nn.BatchNorm2d(filters // 4)\n        self.deconv2 = nn.ConvTranspose2d(filters // 4, filters // 2, kernel_size=kernel, stride=2, dilation=1,\n                                          output_padding=1, padding=1)\n        self.bn2 = nn.BatchNorm2d(filters // 2)\n        self.deconv3 = nn.ConvTranspose2d(filters // 2, channels, kernel_size=kernel, stride=2, dilation=1,\n                                          output_padding=1, padding=1)\n        self.bn3 = nn.BatchNorm2d(channels)\n\n    def forward(self, x):\n        x = self.bn1(self.relu(self.deconv1(x)))\n        x = self.bn2(self.relu(self.deconv2(x)))\n        x = self.bn3(self.relu(self.deconv3(x)))\n\n        return torch.sigmoid(x)\n\n    \nclass ConvVariationalAutoEncoder(nn.Module):\n\n    def __init__(self, channels, filters, embeddings, kernel):\n        super(ConvVariationalAutoEncoder, self).__init__()\n\n        self.encoder = Encode(channels, filters, embeddings, kernel)\n        self.decoder = Decode(channels, filters, embeddings, kernel)\n\n    def forward(self, x):\n        mu, sigma = self.encoder(x)\n        epsilon = torch.randn_like(sigma)\n\n        z_new = mu + sigma * epsilon\n        x_constructed = self.decoder(z_new)\n\n        return x_constructed, mu, sigma","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:45:21.394755Z","iopub.execute_input":"2023-10-21T14:45:21.395475Z","iopub.status.idle":"2023-10-21T14:45:21.412804Z","shell.execute_reply.started":"2023-10-21T14:45:21.395438Z","shell.execute_reply":"2023-10-21T14:45:21.411847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ConvVariationalAutoEncoder(1, 256, 16, 3)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:45:23.812315Z","iopub.execute_input":"2023-10-21T14:45:23.813171Z","iopub.status.idle":"2023-10-21T14:45:23.824352Z","shell.execute_reply.started":"2023-10-21T14:45:23.813137Z","shell.execute_reply":"2023-10-21T14:45:23.823519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model.cuda(), input_size=(1, 28, 28), batch_size=-1)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:45:25.670903Z","iopub.execute_input":"2023-10-21T14:45:25.671618Z","iopub.status.idle":"2023-10-21T14:45:25.684956Z","shell.execute_reply.started":"2023-10-21T14:45:25.671583Z","shell.execute_reply":"2023-10-21T14:45:25.684013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_graph = draw_graph(model.cuda(), input_size=(1, 1, 28, 28), expand_nested=True)\nmodel_graph.visual_graph","metadata":{"execution":{"iopub.status.busy":"2023-10-21T15:06:35.061416Z","iopub.execute_input":"2023-10-21T15:06:35.062264Z","iopub.status.idle":"2023-10-21T15:06:35.255862Z","shell.execute_reply.started":"2023-10-21T15:06:35.062234Z","shell.execute_reply":"2023-10-21T15:06:35.254881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_func = nn.BCELoss()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:46:19.300564Z","iopub.execute_input":"2023-10-21T14:46:19.301302Z","iopub.status.idle":"2023-10-21T14:46:19.305508Z","shell.execute_reply.started":"2023-10-21T14:46:19.301270Z","shell.execute_reply":"2023-10-21T14:46:19.304490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(iter(train_dataloader))\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=[13, 14], dpi=100)\naxes = axes.ravel()\n\nfor i, (img, label) in enumerate(zip(images, labels)):\n    axes[i].imshow(img.numpy()[0], cmap=plt.cm.gray)\n#     axes[i].set_title(label.numpy())\n    \n    if i >= 15:\n        break\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:46:23.273149Z","iopub.execute_input":"2023-10-21T14:46:23.273536Z","iopub.status.idle":"2023-10-21T14:46:25.580168Z","shell.execute_reply.started":"2023-10-21T14:46:23.273508Z","shell.execute_reply":"2023-10-21T14:46:25.579225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:46:31.980114Z","iopub.execute_input":"2023-10-21T14:46:31.980497Z","iopub.status.idle":"2023-10-21T14:46:31.986864Z","shell.execute_reply.started":"2023-10-21T14:46:31.980467Z","shell.execute_reply":"2023-10-21T14:46:31.985908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Training model","metadata":{}},{"cell_type":"code","source":"optimizer = torch.optim.AdamW(model.parameters(), lr=0.0001)\n# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs//8)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:46:43.372216Z","iopub.execute_input":"2023-10-21T14:46:43.373109Z","iopub.status.idle":"2023-10-21T14:46:43.378044Z","shell.execute_reply.started":"2023-10-21T14:46:43.373075Z","shell.execute_reply":"2023-10-21T14:46:43.376956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seq_results = train_loop(model, \n                         loss_func,\n                         train_loader=train_dataloader,\n                         val_loader=test_dataloader,\n                         optimizer=optimizer,\n                         lr_schedule=scheduler,\n                         device=device, \n                         epochs=50\n                        )","metadata":{"execution":{"iopub.status.busy":"2023-10-21T14:46:50.591899Z","iopub.execute_input":"2023-10-21T14:46:50.592620Z","iopub.status.idle":"2023-10-21T14:50:03.721165Z","shell.execute_reply.started":"2023-10-21T14:46:50.592586Z","shell.execute_reply":"2023-10-21T14:50:03.719786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=[12, 6], dpi=200)\nsns.lineplot(x='epoch', y='train loss', data=seq_results, label='train')\nsns.lineplot(x='epoch', y='val loss', data=seq_results, label='validation')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-15T10:42:44.070954Z","iopub.execute_input":"2023-10-15T10:42:44.072005Z","iopub.status.idle":"2023-10-15T10:42:44.452862Z","shell.execute_reply.started":"2023-10-15T10:42:44.071964Z","shell.execute_reply":"2023-10-15T10:42:44.451943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Evaluation","metadata":{}},{"cell_type":"code","source":"projected = []\ndecoded = []\n\nencoder = model.enc\ndecoder = model.dec\nbatch = next(iter(test_dataloader))[0].cpu()\n\nencoder = encoder.eval()\nencoder = encoder.cpu()\ndecoder = decoder.eval()\ndecoder = decoder.cpu()\n\nwith torch.no_grad():\n    projected.append(encoder(batch))\n    \nwith torch.no_grad():\n    decoded.append(decoder(projected[0]))","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:03:03.515233Z","iopub.execute_input":"2023-10-15T12:03:03.516206Z","iopub.status.idle":"2023-10-15T12:03:03.610079Z","shell.execute_reply.started":"2023-10-15T12:03:03.516164Z","shell.execute_reply":"2023-10-15T12:03:03.609129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(projected), projected[0].shape, len(decoded), decoded[0].shape","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:03:29.044061Z","iopub.execute_input":"2023-10-15T12:03:29.045070Z","iopub.status.idle":"2023-10-15T12:03:29.051708Z","shell.execute_reply.started":"2023-10-15T12:03:29.045026Z","shell.execute_reply":"2023-10-15T12:03:29.050767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(nrows=4, ncols=4, figsize=[7, 7], dpi=100)\naxes = axes.ravel()\n\nfor i in range(16):\n    axes[i].imshow(projected[0][0][i])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:03:36.175925Z","iopub.execute_input":"2023-10-15T12:03:36.176901Z","iopub.status.idle":"2023-10-15T12:03:37.635589Z","shell.execute_reply.started":"2023-10-15T12:03:36.176858Z","shell.execute_reply":"2023-10-15T12:03:37.634666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(decoded[0][0][0])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:05:48.335790Z","iopub.execute_input":"2023-10-15T12:05:48.336138Z","iopub.status.idle":"2023-10-15T12:05:48.568530Z","shell.execute_reply.started":"2023-10-15T12:05:48.336111Z","shell.execute_reply":"2023-10-15T12:05:48.567594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rand_emd = torch.rand(1, 16, 3, 3)\n\nwith torch.no_grad():\n    recreate = decoder(rand_emd)\n\nfig, axes = plt.subplots(nrows=4, ncols=4, figsize=[7, 7], dpi=100)\naxes = axes.ravel()\n\nfor i in range(16):\n    axes[i].imshow(rand_emd[0][i])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:22:55.596313Z","iopub.execute_input":"2023-10-15T12:22:55.596647Z","iopub.status.idle":"2023-10-15T12:22:57.003461Z","shell.execute_reply.started":"2023-10-15T12:22:55.596622Z","shell.execute_reply":"2023-10-15T12:22:57.002538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(recreate[0][0])","metadata":{"execution":{"iopub.status.busy":"2023-10-15T12:23:01.375596Z","iopub.execute_input":"2023-10-15T12:23:01.375931Z","iopub.status.idle":"2023-10-15T12:23:01.729456Z","shell.execute_reply.started":"2023-10-15T12:23:01.375905Z","shell.execute_reply":"2023-10-15T12:23:01.725790Z"},"trusted":true},"execution_count":null,"outputs":[]}]}