{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\n\nfrom PIL import Image\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport tqdm\nfrom torch.utils import data\n\nprint(\"CUDA Available\", torch.cuda.is_available())\ncuda = torch.cuda.is_available()\nif cuda:  \n  dev = \"cuda:0\" \nelse:  \n  dev = \"cpu\" \n\ndevice = torch.device(dev)  ","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:42:41.132558Z","iopub.execute_input":"2021-05-25T20:42:41.132973Z","iopub.status.idle":"2021-05-25T20:42:42.407441Z","shell.execute_reply.started":"2021-05-25T20:42:41.132874Z","shell.execute_reply":"2021-05-25T20:42:42.405935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tqdm\nbase_path = '/kaggle/input/11-785-fall-20-homework-2-part-2/classification_data'\ntrain_path = os.path.join(base_path, 'train_data')  \nval_path = os.path.join(base_path, 'val_data')  \ntest_path = os.path.join(base_path, 'test_data') # Not used yet\n\nclass FaceDataset():\n    def __init__(self, root, load_all=False):\n            self.root = root\n            self.unique_label_ids = os.listdir(self.root)\n            print(\"Unique labels\", len(self.unique_label_ids))\n            \n            self.image_paths = []\n            self.label_nums = []\n            self.label_ids = []\n            self.num_classes = len(self.unique_label_ids) \n            self.unique_label_nums = list(range(self.num_classes))\n            \n            # Iterate over each folder, which contains the images\n            # from a same ID (a same person).\n            for label_num, label_id in enumerate(self.unique_label_ids):\n                folder_path = os.path.join(self.root, label_id)\n                \n                # Input list\n                images_list = os.listdir(folder_path)\n                images_list_abs = [os.path.join(folder_path, image_path) for image_path in images_list]\n                self.image_paths += images_list_abs\n                \n                # Labels list\n                num_images = len(images_list)\n                image_label_ids = [label_id] * num_images\n                self.label_ids += image_label_ids\n                \n                # We need the index of the label\n                image_label_nums = [label_num] * num_images\n                self.label_nums += image_label_nums\n                \n            assert len(self.image_paths) == len(self.label_nums)\n            \n            self.data_len = len(self.label_nums)\n            print(\"Total samples:\", self.data_len)\n            self.images = dict()\n            if load_all:\n                print(\"Loading all images...\")\n                for i in tqdm.notebook.tqdm(range(self.data_len)):\n                    self.images[i] = self.__load_image(i)\n                \n    def __load_image(self, index):\n        x = Image.open(self.image_paths[index])\n        x = np.array(x).transpose((2,0,1)) # Bring input channel to first dim\n        return torch.Tensor(x)\n    \n    def __len__(self):\n        return self.data_len\n    \n    def __getitem__(self, index):\n        \"\"\" Lazy load image\"\"\"\n        # TODO: see what to do with this when using all data\n        try:\n            x = self.images[index]\n        except KeyError:\n            x = self.__load_image(index)\n            \n        return x, self.label_nums[index]\n    \n    def reduce_samples(self, ids):\n        current_len = self.data_len\n        new_image_paths = []\n        new_label_nums = []\n        new_label_ids = []\n        \n        for i in range(current_len):\n            label_id = self.label_ids[i]\n            image_path = self.image_paths[i]\n            \n            if label_id in ids:\n                new_label_ids.append(label_id)\n                new_image_paths.append(image_path)\n                new_label_nums.append(ids.index(label_id)) \n        \n        # Update Labels\n        self.label_ids = new_label_ids\n        self.label_nums = new_label_nums\n        self.unique_label_ids = ids\n        self.num_classes = len(self.unique_label_ids)\n        self.unique_label_nums = list(range(self.num_classes))\n        \n        \n        # Update Input\n        self.image_paths = new_image_paths\n        self.images = dict() # Reset lazy loading\n        \n        self.data_len = len(new_label_ids)\n        \n        print(\"Before\", current_len)\n        print(\"After\", self.data_len)\n        print(\"Label nums\", len(self.label_nums))\n        print(\"New Unique labels\", self.num_classes, self.unique_label_nums)   \n    ","metadata":{"execution":{"iopub.status.busy":"2021-05-25T20:42:50.307485Z","iopub.execute_input":"2021-05-25T20:42:50.307800Z","iopub.status.idle":"2021-05-25T20:42:50.326794Z","shell.execute_reply.started":"2021-05-25T20:42:50.307773Z","shell.execute_reply":"2021-05-25T20:42:50.325963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define parameters and hyper parameters","metadata":{}},{"cell_type":"code","source":"params = dict(\n    epochs=10,\n    batch_size=128,\n    num_labels=300,\n    criterion=nn.CrossEntropyLoss,\n    optimizer=torch.optim.SGD,\n    optimizer_params=dict(\n        lr=0.1,\n        #momentum=0.9\n        #    weight_decay=0.01\n    )\n)\n\nIN_CHANNELS = 3","metadata":{"execution":{"iopub.status.busy":"2021-05-25T21:30:56.419499Z","iopub.execute_input":"2021-05-25T21:30:56.419805Z","iopub.status.idle":"2021-05-25T21:30:56.423987Z","shell.execute_reply.started":"2021-05-25T21:30:56.419777Z","shell.execute_reply":"2021-05-25T21:30:56.423146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Instantiate dataset","metadata":{}},{"cell_type":"code","source":"print(\"Training Data\")\ntrain_dataset = FaceDataset(train_path)\nlabels = train_dataset.unique_label_ids[:params['num_labels']]\ntrain_dataset.reduce_samples(labels)\nprint()\nprint(\"Validation Data\")\nval_dataset = FaceDataset(val_path)\nval_dataset.reduce_samples(labels)\n\n\ntrain_loader_args = dict(shuffle=True,\n                        batch_size=params['batch_size'],               \n                        )\ntrain_loader = data.DataLoader(train_dataset, **train_loader_args)\nval_loader = data.DataLoader(val_dataset, **train_loader_args)\nNUM_CLASSES = train_dataset.num_classes\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T21:30:56.428694Z","iopub.execute_input":"2021-05-25T21:30:56.429029Z","iopub.status.idle":"2021-05-25T21:31:03.002999Z","shell.execute_reply.started":"2021-05-25T21:30:56.429004Z","shell.execute_reply":"2021-05-25T21:31:03.002153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_dataset.image_paths[:2])\nprint(train_dataset.label_nums[:2])\ntrain_dataset[0]\nprint(len(train_dataset))\n\n# Just checking\nfrom itertools import groupby\na = [len(list(group)) for key, group in groupby(train_dataset.label_nums)]\ndirectories = [os.path.dirname(path) for path in train_dataset.image_paths]\nb = [len(list(group)) for key, group in groupby(directories)]\n\nassert a == b","metadata":{"execution":{"iopub.status.busy":"2021-05-25T21:31:03.004441Z","iopub.execute_input":"2021-05-25T21:31:03.004946Z","iopub.status.idle":"2021-05-25T21:31:03.054191Z","shell.execute_reply.started":"2021-05-25T21:31:03.004891Z","shell.execute_reply":"2021-05-25T21:31:03.053441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"import tqdm\n\nclass MyModel(nn.Module):\n    def __init__(self, in_channels, out_size):\n        super().__init__()\n        \n        layers = [\n            nn.Conv2d(in_channels, out_channels=24, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.Conv2d(24, out_channels=24, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.MaxPool2d(2, 2),\n            \n            nn.Conv2d(24, 12, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.Conv2d(12, 12, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.MaxPool2d(2, 2),\n            \n            nn.Conv2d(12, 6, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.Conv2d(6, 6, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.Conv2d(6, 6, kernel_size=3, stride=1, padding=1),\n            nn.PReLU(),\n            nn.MaxPool2d(2, 2),\n            \n            nn.Sigmoid(),\n            nn.Flatten()        \n        ]\n        \n        out_shape = (8*8*6, out_size)\n        linear_layer = nn.Linear(*out_shape)\n        layers.append(linear_layer)\n        \n        self.layers = nn.Sequential(*layers)\n        \n    def forward(self, x):        \n        return self.layers(x)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T22:19:30.888399Z","iopub.execute_input":"2021-05-25T22:19:30.888720Z","iopub.status.idle":"2021-05-25T22:19:30.900945Z","shell.execute_reply.started":"2021-05-25T22:19:30.888692Z","shell.execute_reply":"2021-05-25T22:19:30.900125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\n\ndef validate(model, criterion, dev_loader, verbose=False):\n    model.eval()\n    \n    num_correct = 0\n    total_samples = 0\n    losses = []\n    if verbose:\n        print(\"Total Label nums\", len(dev_loader.dataset.label_nums))\n    for x_batch, y_batch in dev_loader:\n        x_batch = x_batch.to(device)\n        y_batch = y_batch.to(device)\n    \n        out = model(x_batch)\n        loss = criterion(out, y_batch)\n        losses.append(loss.item())\n        preds = torch.argmax(out, axis=1)\n        if verbose:\n            print(\"Preds\", preds)\n            print(\"Desired out\", y_batch)\n        \n        # assert all(item in dev_loader.dataset.unique_label_nums for item in preds)\n        correct = int((preds == y_batch).sum())\n        num_correct += correct\n        total_samples += len(y_batch)\n        \n    accuracy = num_correct / total_samples\n    losses = np.asarray(losses)\n    avg_loss = np.average(losses)\n    \n    return accuracy, avg_loss\n\ndef train(model, optimizer, criterion, train_loader, dev_loader, num_epochs=3):\n    val_accuracies = []\n    train_accuracies = []\n    val_avg_losses = []\n    train_avg_losses = []\n    training_times = []\n    \n\n    \n    for epoch in range(num_epochs):\n        print(\"==== Epoch\", epoch, \" ====\")\n        start = time.time()\n        model.train() # So that it saves the gradients\n        for i, (x_batch, y_batch) in enumerate(tqdm.notebook.tqdm(train_loader)):\n            optimizer.zero_grad() # Remove active gradients\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n        \n            output = model(x_batch)\n            loss = criterion(output, y_batch)\n        \n            loss.backward() # Generate gradients w.r.t the loss\n            optimizer.step() # Update weights of the model using the computed gradients\n        \n        end = time.time()\n        training_time = end - start\n        training_times.append(training_time)\n        print(\"Training time\", training_time)\n        \n        val_accuracy, val_avg_loss  = validate(model, criterion, dev_loader)\n        val_accuracies.append(val_accuracy)\n        val_avg_losses.append(val_avg_loss)\n        print(\"Validation Accuracy\", val_accuracy)\n        print(\"Validation Average Loss\", val_avg_loss)\n        \n        train_accuracy, train_avg_loss = validate(model, criterion, train_loader)\n        train_accuracies.append(train_accuracy)\n        train_avg_losses.append(train_avg_loss)\n        print(\"Train Accuracy\", train_accuracy)\n        print(\"Train Average Loss\", train_avg_loss)\n    \n    return val_accuracies, val_avg_losses, train_accuracies, train_avg_losses, training_times","metadata":{"execution":{"iopub.status.busy":"2021-05-25T22:19:32.882777Z","iopub.execute_input":"2021-05-25T22:19:32.883138Z","iopub.status.idle":"2021-05-25T22:19:33.037152Z","shell.execute_reply.started":"2021-05-25T22:19:32.883104Z","shell.execute_reply":"2021-05-25T22:19:33.035593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\nmodel = MyModel(IN_CHANNELS, NUM_CLASSES)\nmodel = model.to(device)\ncriterion = params['criterion']()\noptimizer = params['optimizer'](model.parameters(), **params['optimizer_params'])\n\nnum_params = count_parameters(model)\nprint(model)\nprint(\"Number of parameters\", num_params)\nprint(criterion)\nprint(optimizer)\n\nval_accuracies, val_avg_losses, train_accuracies, train_avg_losses, training_times = train(model, optimizer, criterion, train_loader, val_loader, params['epochs'])\nprint(\"Training Times\", training_times[:2])","metadata":{"execution":{"iopub.status.busy":"2021-05-25T22:19:36.134479Z","iopub.execute_input":"2021-05-25T22:19:36.134779Z","iopub.status.idle":"2021-05-25T22:28:17.893709Z","shell.execute_reply.started":"2021-05-25T22:19:36.134751Z","shell.execute_reply":"2021-05-25T22:28:17.892830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\ndef visualize_results(val_accuracies, val_losses, train_accuracies, train_losses):\n    \n    plt.style.use('ggplot')\n    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(12,6), dpi=80)\n    \n    ax1.plot(val_accuracies, label='Validation Accuracy', marker='o')\n    ax1.plot(train_accuracies, label='Training Accuracy', marker='x')\n    ax1.set_ylabel('Accuracy')\n    ax1.set_xlabel('Epoch')\n    ax1.set_title('Accuracy vs. Epochs')\n    ax1.legend()\n    \n    ax2.plot(val_losses, label='Validation Loss', marker='o')\n    ax2.plot(train_losses, label='Training Loss', marker='x')\n    ax2.set_ylabel('Loss')\n    ax2.set_xlabel('Epoch')\n    ax2.set_title('Loss vs. Epochs')\n    ax2.legend()\n\n    plt.show()\n\nvisualize_results(val_accuracies, val_avg_losses, train_accuracies, train_avg_losses)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T22:28:25.860273Z","iopub.execute_input":"2021-05-25T22:28:25.860578Z","iopub.status.idle":"2021-05-25T22:28:26.136935Z","shell.execute_reply.started":"2021-05-25T22:28:25.860550Z","shell.execute_reply":"2021-05-25T22:28:26.136188Z"},"trusted":true},"execution_count":null,"outputs":[]}]}