{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Overview\nIn this notebook, I built a basic convolutional neural network, LeNet and AlexNet using PyTorch on CIFAR data. <br>\nThis may be helpful for those who just completed a course regarding CNN but do not know where to go next. <br>\nAs a side note, before implementing this, I went through the following courses before working on this. <br>\n1. [DeepLearning.AI Tensorflow Developer Professional Certificate at Coursera](https://www.coursera.org/professional-certificates/tensorflow-in-practice)\n2. [Computer Vision at Kaggle](https://www.kaggle.com/learn/computer-vision)\n3. [PyTorch for Deep Learning with Python Bootcamp at Udemy](https://www.udemy.com/course/pytorch-for-deep-learning-with-python-bootcamp/)\n4. [PyTorch for Deep Learning and Computer Vision at Udemy](https://www.udemy.com/course/pytorch-for-deep-learning-and-computer-vision/)\n\nYou may refer to [my previous notebook](https://www.kaggle.com/code/norifumiirie/simple-neural-network-pytorch-udemy-adoption) where I implemented a CNN likewise on MNIST data according to what I learned through the relevant Udemy courses.","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torchvision import datasets, transforms, models\nfrom torch.utils.data import DataLoader, Dataset, Subset\nfrom sklearn import preprocessing","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-21T06:31:25.089474Z","iopub.execute_input":"2022-12-21T06:31:25.089788Z","iopub.status.idle":"2022-12-21T06:31:28.180292Z","shell.execute_reply.started":"2022-12-21T06:31:25.089714Z","shell.execute_reply":"2022-12-21T06:31:28.178035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Turn on the training that you want to proceed\nUNZIP = False\nPICKLED_DATA_USE = True\n\nCONVNET_TRAINING = False\nCONVNET_PREDICTION = False\nLENET_TRAINING = False\nLENET_PREDICTION = False\nALEXNET_TRAINING = True\nALEXNET_PREDICTION = True","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:28.182442Z","iopub.execute_input":"2022-12-21T06:31:28.183581Z","iopub.status.idle":"2022-12-21T06:31:28.203997Z","shell.execute_reply.started":"2022-12-21T06:31:28.183531Z","shell.execute_reply":"2022-12-21T06:31:28.200484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Constant Variables\nDATA_DIR = \"/kaggle/input/cifar-10/\"\nPICKLED_DATA_DIR = \"/kaggle/input/the-cifar10-dataset/cifar-10-batches-py/\"\n\n# For Google Colab\n# DATA_DIR = \"\"\nSUBMISSION = os.path.join(DATA_DIR,\"sampleSubmission.csv\")\nTRAIN = os.path.join(DATA_DIR,\"train.7z\")\nTRAIN_LABEL = os.path.join(DATA_DIR, \"trainLabels.csv\")\nTEST = os.path.join(DATA_DIR,\"test.7z\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:28.210291Z","iopub.execute_input":"2022-12-21T06:31:28.212418Z","iopub.status.idle":"2022-12-21T06:31:28.232486Z","shell.execute_reply.started":"2022-12-21T06:31:28.212355Z","shell.execute_reply":"2022-12-21T06:31:28.231454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    # py7zr is required to extract the .7tz files\n    !pip install -q py7zr\n\n    from py7zr import unpack_7zarchive\n    import shutil\n    shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)\nexcept Exception as e:\n    print(e)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:28.240848Z","iopub.execute_input":"2022-12-21T06:31:28.241616Z","iopub.status.idle":"2022-12-21T06:31:43.616270Z","shell.execute_reply.started":"2022-12-21T06:31:28.241580Z","shell.execute_reply":"2022-12-21T06:31:43.615173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Unzipping train files and loading them are very memory consuming.\nif UNZIP:\n    # unzip train data\n    shutil.unpack_archive(TRAIN)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:43.618146Z","iopub.execute_input":"2022-12-21T06:31:43.618544Z","iopub.status.idle":"2022-12-21T06:31:43.626016Z","shell.execute_reply.started":"2022-12-21T06:31:43.618505Z","shell.execute_reply":"2022-12-21T06:31:43.624353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The cell above exhausts CPU and takes around 15 mins to complete the unpacking.\nSo, I opted for a [ready-made pickled dataset](https://www.kaggle.com/datasets/ekaakurniawan/the-cifar10-dataset) and turned off the unzipping process.","metadata":{}},{"cell_type":"code","source":"def load_cifar10_data(filename):\n    with open(PICKLED_DATA_DIR + filename, 'rb') as file:\n        batch = pickle.load(file, encoding='latin1')\n        features = batch['data']\n        labels = batch['labels']\n        filenames = batch['filenames']\n        return features, labels, filenames","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:43.627303Z","iopub.execute_input":"2022-12-21T06:31:43.627742Z","iopub.status.idle":"2022-12-21T06:31:43.638454Z","shell.execute_reply.started":"2022-12-21T06:31:43.627708Z","shell.execute_reply":"2022-12-21T06:31:43.637452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_cifar10_meta_data(filename):\n    with open(PICKLED_DATA_DIR + filename, 'rb') as file:\n        batch = pickle.load(file, encoding='latin1')\n        labels = batch['label_names']\n        return labels","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:43.639677Z","iopub.execute_input":"2022-12-21T06:31:43.640081Z","iopub.status.idle":"2022-12-21T06:31:43.649947Z","shell.execute_reply.started":"2022-12-21T06:31:43.640047Z","shell.execute_reply":"2022-12-21T06:31:43.648921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Load files\ntrain_1, train_labels_1, train_filenames_1 = load_cifar10_data('data_batch_1')\ntrain_2, train_labels_2, train_filenames_2 = load_cifar10_data('data_batch_2')\ntrain_3, train_labels_3, train_filenames_3 = load_cifar10_data('data_batch_3')\ntrain_4, train_labels_4, train_filenames_4 = load_cifar10_data('data_batch_4')\ntrain_5, train_labels_5, train_filenames_5 = load_cifar10_data('data_batch_5')\nmeta = load_cifar10_meta_data('batches.meta')\n\n# test, test_labels, test_filenames = load_cifar10_data('test_batch')\n\n# Merge files\nX_train = np.concatenate([train_1, train_2, train_3, train_4, train_5], 0)\ny_train = np.concatenate([train_labels_1, train_labels_2, train_labels_3, train_labels_4, train_labels_5], 0)\n\nprint(meta)\nprint(X_train)\nprint(y_train)\n\n# Wall time was approximately 0.200 seconds.","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:43.651236Z","iopub.execute_input":"2022-12-21T06:31:43.651705Z","iopub.status.idle":"2022-12-21T06:31:48.295114Z","shell.execute_reply.started":"2022-12-21T06:31:43.651669Z","shell.execute_reply":"2022-12-21T06:31:48.294053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = np.vstack(X_train).reshape(-1, 3, 32, 32).transpose((0, 2, 3, 1))\nprint(X_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.296624Z","iopub.execute_input":"2022-12-21T06:31:48.296961Z","iopub.status.idle":"2022-12-21T06:31:48.421170Z","shell.execute_reply.started":"2022-12-21T06:31:48.296925Z","shell.execute_reply":"2022-12-21T06:31:48.420108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a method to load train data just in case\ndef load_train_data():\n    # Load files\n    train_1, train_labels_1, train_filenames_1 = load_cifar10_data('data_batch_1')\n    train_2, train_labels_2, train_filenames_2 = load_cifar10_data('data_batch_2')\n    train_3, train_labels_3, train_filenames_3 = load_cifar10_data('data_batch_3')\n    train_4, train_labels_4, train_filenames_4 = load_cifar10_data('data_batch_4')\n    train_5, train_labels_5, train_filenames_5 = load_cifar10_data('data_batch_5')\n    meta = load_cifar10_meta_data('batches.meta')\n\n    # test, test_labels, test_filenames = load_cifar10_data('test_batch')\n\n    # Merge files\n    X_train = np.concatenate([train_1, train_2, train_3, train_4, train_5], 0)\n    y_train = np.concatenate([train_labels_1, train_labels_2, train_labels_3, train_labels_4, train_labels_5], 0)\n    \n    X_train = np.vstack(X_train).reshape(-1, 3, 32, 32).transpose((0, 2, 3, 1))\n    print(X_train.shape)\n    \n    return X_train, y_train","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.426097Z","iopub.execute_input":"2022-12-21T06:31:48.426381Z","iopub.status.idle":"2022-12-21T06:31:48.437149Z","shell.execute_reply.started":"2022-12-21T06:31:48.426354Z","shell.execute_reply":"2022-12-21T06:31:48.436117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Quick Check on How Each Data Looks Like","metadata":{}},{"cell_type":"markdown","source":"Conventionally, an image can be loaded as `PILimage` in python. <br>\nTherefore, it is required to load an image as `PILimage` first and convert it to `np.array` normally.\nSince I use the pickled dataset, the conversion can be skipped.","metadata":{}},{"cell_type":"code","source":"if UNZIP:\n    print(\"Original data is extracted.\")\n    TRAIN = './train'\n    first_image_id = '1'\n    file_extension = '.png'\n    # Load the first image\n    first_image = Image.open(TRAIN + '/' + first_image_id + file_extension)\n    # Convert the first image to numpy array\n    first_image_arr = np.asarray(first_image)\n    # Check the shape\n    print(first_image_arr.shape)\n    \nif PICKLED_DATA_USE:\n    print(\"Pickled data is extracted.\")\n    first_image_arr = X_train[0]\n    print(first_image_arr.shape) # This results in 3072 as the data is flattened.","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.438508Z","iopub.execute_input":"2022-12-21T06:31:48.438884Z","iopub.status.idle":"2022-12-21T06:31:48.449096Z","shell.execute_reply.started":"2022-12-21T06:31:48.438840Z","shell.execute_reply":"2022-12-21T06:31:48.448125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"That tells you that each image is 32 pixcel by 32 pixcel with 3 color channels.","metadata":{}},{"cell_type":"markdown","source":"Next, before displaying an image, get its label.","metadata":{}},{"cell_type":"code","source":"if UNZIP:\n    y_train_df = pd.read_csv(TRAIN_LABEL)\n    print(len(y_train_df)) # Print the size of the training data\n    display(y_train_df.head())\n    # Get the label of the first image\n    first_image_label = y_train_df.to_numpy()[0][1]\n    print(first_image_label)\n    \nif PICKLED_DATA_USE:\n    first_image_label_index = y_train[0]\n    first_image_label = meta[first_image_label_index]\n    print(first_image_label)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.451468Z","iopub.execute_input":"2022-12-21T06:31:48.451737Z","iopub.status.idle":"2022-12-21T06:31:48.462681Z","shell.execute_reply.started":"2022-12-21T06:31:48.451713Z","shell.execute_reply":"2022-12-21T06:31:48.461624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Everything is ready. Let's display the first image with the obtained label.","metadata":{}},{"cell_type":"code","source":"# Display the first image\nchannel_count = 3\n\nif first_image_arr.shape[2] != channel_count:\n    first_image_arr = np.moveaxis(first_image_arr, 2, 0)\n    print('Shape of the training data after axis conversion:')\n\nprint(first_image_arr.shape)\n    \nplt.imshow(first_image_arr);\nplt.title(first_image_label);","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.464942Z","iopub.execute_input":"2022-12-21T06:31:48.465311Z","iopub.status.idle":"2022-12-21T06:31:48.714341Z","shell.execute_reply.started":"2022-12-21T06:31:48.465277Z","shell.execute_reply":"2022-12-21T06:31:48.713376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Defining CustomDataset Class\nYou can use [datasets.ImageFolder from torchvision](https://ryanwingate.com/intro-to-machine-learning/deep-learning-with-pytorch/loading-image-data-into-pytorch/) if images are classified to subfolders each of which is named as a label. <br>\nUnfortunately, that is not a case here. So, you would extend the dataset class and define your own custom dataset using the given template below.\n\n```\nclass CustomImageDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])\n        image = read_image(img_path)\n        label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        return image, label\n```\n\nRef: https://pytorch.org/tutorials/beginner/basics/data_tutorial.html#creating-a-custom-dataset-for-your-files\n","metadata":{}},{"cell_type":"code","source":"if UNZIP:\n    class CustomImageDataset(Dataset):\n        def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n            self.img_labels = pd.read_csv(annotations_file)\n            self.img_dir = img_dir\n            self.transform = transform\n            self.target_transform = target_transform\n\n        def __len__(self):\n            return len(self.img_labels)\n\n        def __getitem__(self, idx):\n            # Define the path where images are stored by yourself.\n            # Remember, the first image is ./train/1.png\n            img_path = self.img_dir + '/' + str(self.img_labels.iloc[idx, 0]) +'.png'\n\n            # Read image. Recall how you converted an image to numpy array in the previous step.\n            image = np.asarray(Image.open(img_path).convert('RGB'))\n\n            label = self.img_labels.iloc[idx, 1]\n            if self.transform:\n                image = self.transform(image)\n            if self.target_transform:\n                label = self.target_transform(label)\n            return image, label\n\nif PICKLED_DATA_USE:\n    class CustomImageDataset(Dataset):\n        def __init__(self, data, labels, transform=None):\n            self.data = data\n            self.labels = labels\n            self.transform = transform\n\n        def __len__(self):\n            return len(self.data)    \n            \n        def __getitem__(self, index):\n            image, label = self.data[index], self.labels[index]\n\n            if self.transform:\n                image = self.transform(image)\n            \n#             print(image.shape)\n            return image, label","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.715975Z","iopub.execute_input":"2022-12-21T06:31:48.716640Z","iopub.status.idle":"2022-12-21T06:31:48.861437Z","shell.execute_reply.started":"2022-12-21T06:31:48.716591Z","shell.execute_reply":"2022-12-21T06:31:48.860386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if UNZIP:\n    train_dataset = CustomImageDataset(annotations_file = TRAIN_LABEL, img_dir=TRAIN)\n    \nif PICKLED_DATA_USE:\n    train_dataset = CustomImageDataset(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.863184Z","iopub.execute_input":"2022-12-21T06:31:48.863586Z","iopub.status.idle":"2022-12-21T06:31:48.877197Z","shell.execute_reply.started":"2022-12-21T06:31:48.863524Z","shell.execute_reply":"2022-12-21T06:31:48.876188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset = train_dataset, batch_size = 100, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.878966Z","iopub.execute_input":"2022-12-21T06:31:48.879321Z","iopub.status.idle":"2022-12-21T06:31:48.887452Z","shell.execute_reply.started":"2022-12-21T06:31:48.879286Z","shell.execute_reply":"2022-12-21T06:31:48.886358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check if the dataloader loading the dataset object that you defined on your own works by running the cell below.","metadata":{}},{"cell_type":"code","source":"# Fetch the iterator\ntrain_iter = iter(train_loader)\n\n# Get the first batch\nimages, labels = train_iter.next()\n\n# Specify the figure width and height\nfig = plt.figure(figsize=(25, 4))\n\nfor i in range(1, 21):\n    ax = fig.add_subplot(2, 10, i, xticks=[], yticks=[])\n    plt.imshow(images[i-1])\n    ax.set_title(meta[labels[i-1].item()])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:48.888820Z","iopub.execute_input":"2022-12-21T06:31:48.889504Z","iopub.status.idle":"2022-12-21T06:31:49.661900Z","shell.execute_reply.started":"2022-12-21T06:31:48.889469Z","shell.execute_reply":"2022-12-21T06:31:49.660920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images[0].dtype","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.662892Z","iopub.execute_input":"2022-12-21T06:31:49.663242Z","iopub.status.idle":"2022-12-21T06:31:49.670789Z","shell.execute_reply.started":"2022-12-21T06:31:49.663207Z","shell.execute_reply":"2022-12-21T06:31:49.669752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Without any transformation of the data, the dtype should be `torch.uint8`, which cannot be processed by `model.forward`. <br>\nThus, re-define the dataset with a transform method.","metadata":{}},{"cell_type":"code","source":"my_transform = transforms.Compose([\n    transforms.ToPILImage(), # Without this, resizing will end up with an error\n    transforms.ToTensor(), # Convert an image to Tensor\n    transforms.ConvertImageDtype(torch.float32) # Convert dtype of an image to float\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.672423Z","iopub.execute_input":"2022-12-21T06:31:49.673683Z","iopub.status.idle":"2022-12-21T06:31:49.680440Z","shell.execute_reply.started":"2022-12-21T06:31:49.673648Z","shell.execute_reply":"2022-12-21T06:31:49.679376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Also, if the label is `string`. It should be encoded to a numeric value.\nThus, define an encoder and re-define `CustomImageDataset`.","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv(TRAIN_LABEL)\nlabel_list = meta\nprint(label_list)\n\n# Define an encoder\nlabel_encoder = preprocessing.LabelEncoder()   \n# Fit the encoder\nlabel_encoder.fit_transform(label_list)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.683694Z","iopub.execute_input":"2022-12-21T06:31:49.684008Z","iopub.status.idle":"2022-12-21T06:31:49.736428Z","shell.execute_reply.started":"2022-12-21T06:31:49.683980Z","shell.execute_reply":"2022-12-21T06:31:49.735435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if UNZIP:\n    class CustomImageDataset(Dataset):\n        def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n            self.img_labels = pd.read_csv(annotations_file)\n            self.img_dir = img_dir\n            self.transform = transform\n            self.target_transform = target_transform\n\n        def __len__(self):\n            return len(self.img_labels)\n\n        def __getitem__(self, idx):\n            # Define the path where images are stored by yourself.\n            # Remember, the first image is ./train/1.png\n            img_path = self.img_dir + '/' + str(self.img_labels.iloc[idx, 0]) +'.png'\n\n            # Read image. Recall how you converted an image to numpy array in the previous step.\n            image = np.asarray(Image.open(img_path).convert('RGB'))\n\n            label = self.img_labels.iloc[idx, 1] # Due to this, labels are string. You need to encode them.\n            label = label_encoder.transform([label])\n\n            if self.transform:\n                image = self.transform(image)\n            if self.target_transform:\n                label = self.target_transform(label)\n            return image, label","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.737778Z","iopub.execute_input":"2022-12-21T06:31:49.738595Z","iopub.status.idle":"2022-12-21T06:31:49.748051Z","shell.execute_reply.started":"2022-12-21T06:31:49.738556Z","shell.execute_reply":"2022-12-21T06:31:49.747020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if UNZIP:        \n    train_dataset = CustomImageDataset(annotations_file=TRAIN_LABEL, img_dir=TRAIN, transform=my_transform)\n    \nif PICKLED_DATA_USE:\n    train_dataset = CustomImageDataset(X_train, y_train, transform=my_transform)\n\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=500, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.749638Z","iopub.execute_input":"2022-12-21T06:31:49.750453Z","iopub.status.idle":"2022-12-21T06:31:49.760257Z","shell.execute_reply.started":"2022-12-21T06:31:49.750390Z","shell.execute_reply":"2022-12-21T06:31:49.759290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the method below so the weight of each param in a model can be reset\ndef reset_model(model):  \n    for layer in model.children():\n#         print(layer)\n        if hasattr(layer, 'reset_parameters'):\n            layer.reset_parameters()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.761535Z","iopub.execute_input":"2022-12-21T06:31:49.762581Z","iopub.status.idle":"2022-12-21T06:31:49.774431Z","shell.execute_reply.started":"2022-12-21T06:31:49.762544Z","shell.execute_reply":"2022-12-21T06:31:49.773322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ConvNet\nFirstly, I use a CNN model I built for MNIST Dataset in [the previous Notebook](https://www.kaggle.com/code/norifumiirie/simple-neural-network-pytorch-udemy-adoption). The model is adjusted according to the new input shape.","metadata":{}},{"cell_type":"code","source":"class ConvNet(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(\n                in_channels=3,      # 3 Color Channels (RGB)        \n                out_channels=6,            \n                kernel_size=3,              \n                stride=1           \n        )\n        self.conv2 = nn.Conv2d(6, 16, 3, 1)\n        self.linear1 = nn.Linear(16*6*6, 120)\n        self.linear2 = nn.Linear(120, 84)\n        self.linear3 = nn.Linear(84, 10)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2, 2)\n        x = self.conv2(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2, 2)\n        # Next, flatten the predicted value. Careful with the byte size\n        x = x.view(-1, 16*6*6) # Using '-1', the first dimension is retained as 100\n        # Insert some linear layers\n        x = self.linear1(x)\n        x = self.linear2(x)\n        x = self.linear3(x)\n        \n        # Softmax\n        x = F.log_softmax(x, dim=1)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.776030Z","iopub.execute_input":"2022-12-21T06:31:49.777074Z","iopub.status.idle":"2022-12-21T06:31:49.788929Z","shell.execute_reply.started":"2022-12-21T06:31:49.776996Z","shell.execute_reply":"2022-12-21T06:31:49.787926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ConvNet() # Define this for validation purposes\nmodel_on_gpu = ConvNet().cuda() # Define this to actually train the model on the GPU\n\nprint(model)\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_on_gpu.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:49.790407Z","iopub.execute_input":"2022-12-21T06:31:49.790952Z","iopub.status.idle":"2022-12-21T06:31:53.200892Z","shell.execute_reply.started":"2022-12-21T06:31:49.790917Z","shell.execute_reply":"2022-12-21T06:31:53.199804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if model_on_gpu is really on gpu\nnext(model_on_gpu.parameters()).is_cuda","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.203511Z","iopub.execute_input":"2022-12-21T06:31:53.204199Z","iopub.status.idle":"2022-12-21T06:31:53.211353Z","shell.execute_reply.started":"2022-12-21T06:31:53.204159Z","shell.execute_reply":"2022-12-21T06:31:53.210245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before the actual training and prediction, run the cell below to see how the shape of the data changes through the training steps.\n\n","metadata":{}},{"cell_type":"code","source":"for batch_no, (X_train, y_train) in enumerate(train_loader):\n    \n    # PyTorch expects the data to be Channel First. Swap axes if they are not in the right order\n    channel_count = 3\n    \n    if X_train.shape[1] != channel_count:\n        X_train = torch.moveaxis(X_train, 3, 1)\n        print('Shape of the training data after axis conversion:')\n        print(X_train.shape)\n    \n    X_train_for_shape_check = torch.clone(X_train)\n    \n    print('Original shape of the training data:')\n    print(X_train_for_shape_check.shape)\n    print(X_train_for_shape_check.dtype) # If dtype is int, model.forward will not work.\n    \n    # From here, display the shape and image of the data in every step, so further errors can be avoided intuitively while training the model\n    # Relu is just an actiavation function not affecting the image shape, so it will be skipped here.\n    \n    # Specify the figure width and height\n    fig = plt.figure(figsize=(25, 4))\n    \n    X_train_for_shape_check = model.conv1(X_train_for_shape_check)\n    print(X_train_for_shape_check.shape)\n    # After conv1, the data loses the corner info, so the size becomes 30by30\n    \n    # Show the first image after conv1\n    ax = fig.add_subplot(1, 5, 1, xticks=[], yticks=[])\n    plt.title('conv1')\n    plt.imshow(X_train_for_shape_check[0][0].detach().numpy()) \n    \n    X_train_for_shape_check = F.max_pool2d(X_train_for_shape_check, 2, 2)\n    print(X_train_for_shape_check.shape)\n    # After max_pool2d, the data loses the corner info, so the size becomes 15by15\n    \n    # Show the first image after the pooling layer\n    ax = fig.add_subplot(1, 5, 2, xticks=[], yticks=[])\n    plt.title('max_pool2d (1st Time)')\n    plt.imshow(X_train_for_shape_check[0][0].detach().numpy())\n    \n    X_train_for_shape_check = model.conv2(X_train_for_shape_check)\n    print(X_train_for_shape_check.shape)\n    # After conv2, the data loses the corner info, so the size becomes 13by13\n\n    # Show the first image after conv2\n    ax = fig.add_subplot(1, 5, 3, xticks=[], yticks=[])\n    plt.title('conv2')\n    plt.imshow(X_train_for_shape_check[0][0].detach().numpy())\n    \n    X_train_for_shape_check = F.max_pool2d(X_train_for_shape_check, 2, 2)\n    print(X_train_for_shape_check.shape)\n    # After max_pool2d, the data loses the corner info, so the size becomes 6by6\n    \n    # Show the first image after the pooling layer\n    ax = fig.add_subplot(1, 5, 4, xticks=[], yticks=[])\n    plt.title('max_pool2d (2nd Time)')\n    plt.imshow(X_train_for_shape_check[0][0].detach().numpy())    \n\n    # Next, flatten the image data for the fully connected layers\n    X_train_for_shape_check = X_train_for_shape_check.view(-1, 16*6*6) # Using '-1', the first dimension is retained as 100\n    print(X_train_for_shape_check.shape)\n    # After this flattening, the data shape should become torch.Size([500, 576])\n    \n    # If the shape is as expected, run the model.forward method and check if the shape is okay.\n    # Beforehand, reset the data\n    X_train_for_shape_check = torch.clone(X_train)\n    \n    y_pred = model.forward(X_train_for_shape_check)\n    print('Shape after model.forward:')\n    print(y_pred.shape) \n    \n    # Label data (y_train) needs to be flattened from [500,1] to [500] in order to calculate the loss\n    y_train = torch.flatten(y_train)\n    \n    loss = criterion(y_pred, y_train.long()) # y_train has to be LONG\n    \n    optimizer.zero_grad()\n    loss.backward()\n    optimizer.step()\n\n    # Get the predicted numbers\n    _, y_pred = torch.max(y_pred, 1)\n    print(y_pred)\n    \n    break","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.213267Z","iopub.execute_input":"2022-12-21T06:31:53.213685Z","iopub.status.idle":"2022-12-21T06:31:53.874850Z","shell.execute_reply.started":"2022-12-21T06:31:53.213634Z","shell.execute_reply":"2022-12-21T06:31:53.873478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, train the model through epochs. <br>\nUse `model_on_gpu` and move all the tensor data to be used onto the GPU.","metadata":{}},{"cell_type":"code","source":"if CONVNET_TRAINING:\n    start_time = time.time()\n\n    epochs = 20\n    running_loss_history = []\n    running_corrects_history = []\n\n    # To allow u to run this cell multiple times, reset the model first \n    reset_model(model_on_gpu)\n\n    for i in range(epochs):\n\n        running_loss = .0\n        running_corrects = .0\n\n        for batch_no, (X_train, y_train) in enumerate(train_loader):\n            # Increment the batch no.\n            batch_no += 1\n\n            X_train = torch.clone(X_train).cuda() # Move X_train to the GPU\n            y_train = torch.clone(y_train).cuda() # Move y_train to the GPU\n\n            # PyTorch expects the data to be Channel First. Swap axes if they are not in the right order\n            channel_count = 3\n            if X_train.shape[1] != channel_count:\n                X_train = torch.moveaxis(X_train, 3, 1)\n\n            y_pred = model_on_gpu.forward(X_train)\n\n            # Label data (y_train) needs to be flattened from [50,1] to [50] in order to calculate the loss\n            y_train = torch.flatten(y_train)\n\n            loss = criterion(y_pred, y_train.long()) # y_train has to be LONG\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n\n            # Get the predicted numbers\n            _, y_pred = torch.max(y_pred, 1)\n\n            # Get the total number of the correct values\n            running_corrects += torch.sum(y_pred==y_train.data)\n\n        # get the average loss and accuracy\n        epoch_loss = running_loss/50000 \n        running_loss_history.append(epoch_loss)\n        epoch_accuracy = running_corrects.float()/50000 \n        running_corrects_history.append(epoch_accuracy)\n\n        print(f'################## Epoch: {i} ##################')\n        print('Loss: {:.4f}, Accuracy: {:.4f}'.format(epoch_loss, epoch_accuracy))\n        print(f'Duration: {time.time() - start_time:.2f}')  ","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.886960Z","iopub.execute_input":"2022-12-21T06:31:53.887447Z","iopub.status.idle":"2022-12-21T06:31:53.906621Z","shell.execute_reply.started":"2022-12-21T06:31:53.887387Z","shell.execute_reply":"2022-12-21T06:31:53.904987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONVNET_TRAINING:\n    plt.plot(running_loss_history, label='training_loss')","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.908608Z","iopub.execute_input":"2022-12-21T06:31:53.909307Z","iopub.status.idle":"2022-12-21T06:31:53.915874Z","shell.execute_reply.started":"2022-12-21T06:31:53.909270Z","shell.execute_reply":"2022-12-21T06:31:53.915060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Data Processing","metadata":{}},{"cell_type":"markdown","source":"As ConvNet is ready, what I am going to do next is to get the test data ready; it has to be set into `test_loader` using `TestImageDataset`. <br>\nThe pickled `test_batch` is not usable as 290,000 junk images were added to the original CIFAR10 test data for this competition.","metadata":{}},{"cell_type":"code","source":"class TestImageDataset(Dataset):\n    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_dir = img_dir\n        self.transform = transform\n        self.target_transform = target_transform\n\n    def __len__(self):\n        return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        # Define the path where images are stored by yourself.\n        # Remember, the first image is ./train/1.png\n        img_path = self.img_dir + '/' + str(self.img_labels.iloc[idx, 0]) +'.png'\n\n        # Read image. Recall how you converted an image to numpy array in the previous step.\n        image = np.asarray(Image.open(img_path).convert('RGB'))\n\n        label = self.img_labels.iloc[idx, 1]\n        if self.transform:\n            image = self.transform(image)\n        if self.target_transform:\n            label = self.target_transform(label)\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.918753Z","iopub.execute_input":"2022-12-21T06:31:53.919023Z","iopub.status.idle":"2022-12-21T06:31:53.928232Z","shell.execute_reply.started":"2022-12-21T06:31:53.918998Z","shell.execute_reply":"2022-12-21T06:31:53.927561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndir_list = os.listdir()\nif 'test' not in str(dir_list):\n    shutil.unpack_archive(TEST)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:31:53.929852Z","iopub.execute_input":"2022-12-21T06:31:53.930316Z","iopub.status.idle":"2022-12-21T06:46:01.264954Z","shell.execute_reply.started":"2022-12-21T06:31:53.930170Z","shell.execute_reply":"2022-12-21T06:46:01.263809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestImageDataset(annotations_file=SUBMISSION, img_dir='./test', transform=my_transform)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=500, shuffle=False)\n\n# for batch_no, (X_test, _) in enumerate(test_loader):\n#         batch_no += 1\n#         print(batch_no)\n#         print(X_test.shape)\n#         filename = 'test_batch_' + str(batch_no)\n#         with open(filename, 'wb') as f:\n#             pickle.dump(X_test , f)\n\n# Fetch the iterator\ntest_iter = iter(test_loader)\n# Get the first batch\nimages, _ = test_iter.next()\n# Draw the first image\nplt.imshow(torch.moveaxis(images[0], 0, 2))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:01.266596Z","iopub.execute_input":"2022-12-21T06:46:01.267553Z","iopub.status.idle":"2022-12-21T06:46:01.905404Z","shell.execute_reply.started":"2022-12-21T06:46:01.267513Z","shell.execute_reply":"2022-12-21T06:46:01.904335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ConvNet Prediction and Submission","metadata":{}},{"cell_type":"code","source":"# Load and check the sample submission file\nsubmit_df = pd.read_csv(SUBMISSION)\ndisplay(submit_df.head())\nprint(f'Sumbission data size: {len(submit_df)}')","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:01.906850Z","iopub.execute_input":"2022-12-21T06:46:01.907343Z","iopub.status.idle":"2022-12-21T06:46:01.964267Z","shell.execute_reply.started":"2022-12-21T06:46:01.907303Z","shell.execute_reply":"2022-12-21T06:46:01.963124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONVNET_PREDICTION:\n    id_start = 0\n\n    for batch_no, (X_test, _) in enumerate(test_loader):\n\n        batch_size = 500\n\n        id_start = batch_no * batch_size + 1\n        id_end = id_start + batch_size - 1\n\n        # Increment the batch no.\n        batch_no += 1    \n\n        X_test = torch.clone(X_test).cuda() # Move X_train to the GPU\n\n        with torch.no_grad(): # Stop updating the gradient\n            _, y_test_pred = torch.max(model_on_gpu(X_test.float()), 1) # Pick the most probable label\n\n            # Move the predicted values to CPU. This may be heavy for the allocated RAM on the CPU.\n            y_test_pred = y_test_pred.cpu()   \n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # Convert it to numpy\n    #         y_test_pred = y_test_pred.numpy()\n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # As submission expects actual category names such as cat, the predicted values have to be transformed. \n            y_test_pred = label_encoder.inverse_transform(y_test_pred)\n\n            # Reflect the predicted labels for the subjected range in submit_df\n            submit_df.loc[id_start-1:id_end-1, 'label'] = y_test_pred\n\n            if batch_no <= 2: # Print logs until the 2nd iteration\n                print(f' Batch #: {batch_no}')\n                print(f' Submission IDs: {id_start} - {id_end}')\n                print(submit_df[(submit_df.id >= id_start) & (submit_df.id <= id_end)])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:01.966095Z","iopub.execute_input":"2022-12-21T06:46:01.966491Z","iopub.status.idle":"2022-12-21T06:46:01.975427Z","shell.execute_reply.started":"2022-12-21T06:46:01.966454Z","shell.execute_reply":"2022-12-21T06:46:01.974302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CONVNET_PREDICTION:\n    # Check the result before the submission \n    display(submit_df.head())\n    submit_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:01.977571Z","iopub.execute_input":"2022-12-21T06:46:01.977935Z","iopub.status.idle":"2022-12-21T06:46:01.989301Z","shell.execute_reply.started":"2022-12-21T06:46:01.977900Z","shell.execute_reply":"2022-12-21T06:46:01.988433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The submission score was 0.57870.\nAlthough there is still a lot I can do to improve the score such as hyperparamter optimization and data augumentation, I will move to LeNet.","metadata":{}},{"cell_type":"markdown","source":"### LeNet-5","metadata":{}},{"cell_type":"markdown","source":"I am going to build LeNet following Rayan Slim's Udemy course.\nTo try LeNet, I have to built it from scratch as PyTorch is not supporting the original LeNet anyways. <br>\n[Dropout](https://www.thepythoncode.com/article/dropout-regularization-in-pytorch) is one notable feature of LeNet, so you may note it down. <br>\nI will skip hypterparameter optimization and data augumentation to keep the focus of this notebook on model implementations.","metadata":{}},{"cell_type":"code","source":"class LeNet(nn.Module):\n    \n    def __init__(self):\n        super().__init__()\n        input_to_linear_layer = 5*5*64\n        \n        self.conv1 = nn.Conv2d(3, 16, 5, 1)\n        self.conv2 = nn.Conv2d(16, 32, 5, 1)\n        self.conv3 = nn.Conv2d(32, 64, 5, 1)\n        self.linear1 = nn.Linear(input_to_linear_layer, 500)\n        self.dropout1 = nn.Dropout(0.5)\n        self.linear2 = nn.Linear(500, 10)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2, 2)\n        x = self.conv2(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2, 2)\n        x = self.conv3(x)\n        x = F.relu(x)\n        x = F.max_pool2d(x, 2, 2)\n        # Next, flatten the predicted value. Careful with the byte size\n        x = x.view(-1, linear_input) # Using '-1', the first dimension is retained\n        # Insert some linear layers\n        x = self.linear1(x)\n        x = F.relu(x)\n        x = self.dropout1(x)\n        x = self.linear2(x)\n        \n        # Softmax\n        x = F.log_softmax(x, dim=1)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:01.990828Z","iopub.execute_input":"2022-12-21T06:46:01.991229Z","iopub.status.idle":"2022-12-21T06:46:02.002749Z","shell.execute_reply.started":"2022-12-21T06:46:01.991195Z","shell.execute_reply":"2022-12-21T06:46:02.001712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LeNet() # Define this for validation purposes\nmodel_on_gpu = ConvNet().cuda() # Define this to actually train the model on the GPU\n\nprint(model)\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_on_gpu.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.004005Z","iopub.execute_input":"2022-12-21T06:46:02.004439Z","iopub.status.idle":"2022-12-21T06:46:02.029474Z","shell.execute_reply.started":"2022-12-21T06:46:02.004404Z","shell.execute_reply":"2022-12-21T06:46:02.028430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check if model_on_gpu is really on gpu\nnext(model_on_gpu.parameters()).is_cuda","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.030916Z","iopub.execute_input":"2022-12-21T06:46:02.031255Z","iopub.status.idle":"2022-12-21T06:46:02.038944Z","shell.execute_reply.started":"2022-12-21T06:46:02.031221Z","shell.execute_reply":"2022-12-21T06:46:02.037760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, train the model.","metadata":{}},{"cell_type":"code","source":"if LENET_TRAINING:\n    start_time = time.time()\n\n    epochs = 20\n    running_loss_history = []\n    running_corrects_history = []\n\n    # To allow u to run this cell multiple times, reset the model first \n    reset_model(model_on_gpu)\n\n    for i in range(epochs):\n\n        running_loss = .0\n        running_corrects = .0\n\n        for batch_no, (X_train, y_train) in enumerate(train_loader):\n            # Increment the batch no.\n            batch_no += 1\n\n            X_train = torch.clone(X_train).cuda() # Move X_train to the GPU\n            y_train = torch.clone(y_train).cuda() # Move y_train to the GPU\n\n            # PyTorch expects the data to be Channel First. Swap axes if they are not in the right order\n            channel_count = 3\n            if X_train.shape[1] != channel_count:\n                X_train = torch.moveaxis(X_train, 3, 1)\n\n            y_pred = model_on_gpu.forward(X_train)\n\n            # Label data (y_train) needs to be flattened from [50,1] to [50] in order to calculate the loss\n            y_train = torch.flatten(y_train)\n\n            loss = criterion(y_pred, y_train.long()) # y_train has to be LONG\n\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item()\n\n            # Get the predicted numbers\n            _, y_pred = torch.max(y_pred, 1)\n\n            # Get the total number of the correct values\n            running_corrects += torch.sum(y_pred==y_train.data)\n\n        # get the average loss and accuracy\n        epoch_loss = running_loss/50000 \n        running_loss_history.append(epoch_loss)\n        epoch_accuracy = running_corrects.float()/50000 \n        running_corrects_history.append(epoch_accuracy)\n\n        print(f'################## Epoch: {i} ##################')\n        print('Loss: {:.4f}, Accuracy: {:.4f}'.format(epoch_loss, epoch_accuracy))\n        print(f'Duration: {time.time() - start_time:.2f}')  ","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.040665Z","iopub.execute_input":"2022-12-21T06:46:02.041162Z","iopub.status.idle":"2022-12-21T06:46:02.052384Z","shell.execute_reply.started":"2022-12-21T06:46:02.041128Z","shell.execute_reply":"2022-12-21T06:46:02.051372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if LENET_TRAINING:\n    plt.plot(running_loss_history, label='training_loss')","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.053772Z","iopub.execute_input":"2022-12-21T06:46:02.054580Z","iopub.status.idle":"2022-12-21T06:46:02.065230Z","shell.execute_reply.started":"2022-12-21T06:46:02.054541Z","shell.execute_reply":"2022-12-21T06:46:02.064441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### LeNet Prediction and Submission","metadata":{}},{"cell_type":"code","source":"if LENET_PREDICTION:\n    id_start = 0\n\n    for batch_no, (X_test, _) in enumerate(test_loader):\n\n        batch_size = 500\n\n        id_start = batch_no * batch_size + 1\n        id_end = id_start + batch_size - 1\n\n        # Increment the batch no.\n        batch_no += 1    \n\n        X_test = torch.clone(X_test).cuda() # Move X_train to the GPU\n\n        with torch.no_grad(): # Stop updating the gradient\n            _, y_test_pred = torch.max(model_on_gpu(X_test.float()), 1) # Pick the most probable label\n\n            # Move the predicted values to CPU. This may be heavy for the allocated RAM on the CPU.\n            y_test_pred = y_test_pred.cpu()   \n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # Convert it to numpy\n    #         y_test_pred = y_test_pred.numpy()\n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # As submission expects actual category names such as cat, the predicted values have to be transformed. \n            y_test_pred = label_encoder.inverse_transform(y_test_pred)\n\n            # Reflect the predicted labels for the subjected range in submit_df\n            submit_df.loc[id_start-1:id_end-1, 'label_encoded'] = y_test_pred\n\n            if batch_no <= 2: # Print logs until the 2nd iteration\n                print(f' Batch #: {batch_no}')\n                print(f' Submission IDs: {id_start} - {id_end}')\n                print(submit_df[(submit_df.id >= id_start) & (submit_df.id <= id_end)])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.066499Z","iopub.execute_input":"2022-12-21T06:46:02.067474Z","iopub.status.idle":"2022-12-21T06:46:02.076911Z","shell.execute_reply.started":"2022-12-21T06:46:02.067437Z","shell.execute_reply":"2022-12-21T06:46:02.076171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if LENET_PREDICTION:\n    # Check the result before the submission \n    display(submit_df.head())\n    submit_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.077992Z","iopub.execute_input":"2022-12-21T06:46:02.080195Z","iopub.status.idle":"2022-12-21T06:46:02.091653Z","shell.execute_reply.started":"2022-12-21T06:46:02.080169Z","shell.execute_reply":"2022-12-21T06:46:02.090762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The submission score was 0.60050. Slight improvement! But again, although there is still a lot I can do to improve the score such as hyperparamter optimization and data augumentation, now allow me to work on AlexNet next.","metadata":{}},{"cell_type":"markdown","source":"### AlexNet","metadata":{}},{"cell_type":"markdown","source":"To win a Kaggle competition, transfer learning using an established model is a key.\nAlexNet is one of the first ImageNet winning models developed in 2012, which is consdered old.\nLet's use it and see how the score can get higher.","metadata":{}},{"cell_type":"code","source":"model = models.alexnet(pretrained=True)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:02.093045Z","iopub.execute_input":"2022-12-21T06:46:02.093489Z","iopub.status.idle":"2022-12-21T06:46:05.002156Z","shell.execute_reply.started":"2022-12-21T06:46:02.093453Z","shell.execute_reply":"2022-12-21T06:46:05.000377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in model.features.parameters():\n    # Freeze the parameter value\n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.004213Z","iopub.execute_input":"2022-12-21T06:46:05.005296Z","iopub.status.idle":"2022-12-21T06:46:05.010899Z","shell.execute_reply.started":"2022-12-21T06:46:05.005255Z","shell.execute_reply":"2022-12-21T06:46:05.009575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Overwrite the last classifier layer as the output should be only 10\nprint(model.classifier[6])\nn_inputs = model.classifier[6].in_features # 4096\nlast_linear_layer = nn.Linear(n_inputs, 10)\nmodel.classifier[6] = last_linear_layer","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.012500Z","iopub.execute_input":"2022-12-21T06:46:05.013574Z","iopub.status.idle":"2022-12-21T06:46:05.023686Z","shell.execute_reply.started":"2022-12-21T06:46:05.013482Z","shell.execute_reply":"2022-12-21T06:46:05.022723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the overwritten model\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.025014Z","iopub.execute_input":"2022-12-21T06:46:05.025674Z","iopub.status.idle":"2022-12-21T06:46:05.036467Z","shell.execute_reply.started":"2022-12-21T06:46:05.025640Z","shell.execute_reply":"2022-12-21T06:46:05.035376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Properly define an ideal device where the training is done\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.037808Z","iopub.execute_input":"2022-12-21T06:46:05.038407Z","iopub.status.idle":"2022-12-21T06:46:05.046322Z","shell.execute_reply.started":"2022-12-21T06:46:05.038358Z","shell.execute_reply":"2022-12-21T06:46:05.045416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Move the model to the defined device\nmodel_on_gpu = model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.047728Z","iopub.execute_input":"2022-12-21T06:46:05.048281Z","iopub.status.idle":"2022-12-21T06:46:05.122313Z","shell.execute_reply.started":"2022-12-21T06:46:05.048247Z","shell.execute_reply":"2022-12-21T06:46:05.121435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define loss and optimization functions\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_on_gpu.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.123718Z","iopub.execute_input":"2022-12-21T06:46:05.124339Z","iopub.status.idle":"2022-12-21T06:46:05.130097Z","shell.execute_reply.started":"2022-12-21T06:46:05.124304Z","shell.execute_reply":"2022-12-21T06:46:05.129132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, train the model.","metadata":{}},{"cell_type":"code","source":"if ALEXNET_TRAINING:\n    try:\n        start_time = time.time()\n\n        epochs = 20\n        running_loss_history = []\n        running_corrects_history = []\n\n        for i in range(epochs):\n\n            running_loss = .0\n            running_corrects = .0\n\n            for batch_no, (X_train, y_train) in enumerate(train_loader):\n                # Increment the batch no.\n                batch_no += 1\n\n                X_train = torch.clone(X_train).cuda() # Move X_train to the GPU\n                y_train = torch.clone(y_train).cuda() # Move y_train to the GPU\n\n                # PyTorch expects the data to be Channel First. Swap axes if they are not in the right order\n                channel_count = 3\n                if X_train.shape[1] != channel_count:\n                    X_train = torch.moveaxis(X_train, 3, 1)\n\n                y_pred = model_on_gpu.forward(X_train)\n\n                # Label data (y_train) needs to be flattened from [50,1] to [50] in order to calculate the loss\n                y_train = torch.flatten(y_train)\n\n                loss = criterion(y_pred, y_train.long()) # y_train has to be LONG\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                running_loss += loss.item()\n\n                # Get the predicted numbers\n                _, y_pred = torch.max(y_pred, 1)\n\n                # Get the total number of the correct values\n                running_corrects += torch.sum(y_pred==y_train.data)\n\n            # get the average loss and accuracy\n            epoch_loss = running_loss/50000 \n            running_loss_history.append(epoch_loss)\n            epoch_accuracy = running_corrects.float()/50000 \n            running_corrects_history.append(epoch_accuracy)\n\n            print(f'################## Epoch: {i} ##################')\n            print('Loss: {:.4f}, Accuracy: {:.4f}'.format(epoch_loss, epoch_accuracy))\n            print(f'Duration: {time.time() - start_time:.2f}')\n    except Exception as e: print(e)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:05.131511Z","iopub.execute_input":"2022-12-21T06:46:05.132062Z","iopub.status.idle":"2022-12-21T06:46:11.657122Z","shell.execute_reply.started":"2022-12-21T06:46:05.132023Z","shell.execute_reply":"2022-12-21T06:46:11.655685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As the error indicates, CIFAR data is too small (32x32) to dunk into AlexNet.\nI am going to resize the data first.","metadata":{}},{"cell_type":"code","source":"my_transform = transforms.Compose([\n    transforms.ToPILImage(), # Without this, resizing will end up with an error\n    transforms.Resize(256), # the input size of AlexNet is  227∗227 \n    transforms.CenterCrop(224),\n    transforms.ToTensor(), # Convert an image to Tensor\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), # Normalize it just in case\n#     transforms.ConvertImageDtype(torch.float32) # Convert dtype of an image to float\n])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:11.662862Z","iopub.execute_input":"2022-12-21T06:46:11.663630Z","iopub.status.idle":"2022-12-21T06:46:11.677934Z","shell.execute_reply.started":"2022-12-21T06:46:11.663577Z","shell.execute_reply":"2022-12-21T06:46:11.676455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if UNZIP:\n    train_dataset = CustomImageDataset(annotations_file=TRAIN_LABEL, img_dir=TRAIN, transform=my_transform)\nif PICKLED_DATA_USE:\n    X_train, y_train = load_train_data()\n    train_dataset = CustomImageDataset(X_train, y_train, transform=my_transform)\n\ntest_dataset = TestImageDataset(annotations_file=SUBMISSION, img_dir='./test', transform=my_transform)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:11.679795Z","iopub.execute_input":"2022-12-21T06:46:11.684158Z","iopub.status.idle":"2022-12-21T06:46:12.651652Z","shell.execute_reply.started":"2022-12-21T06:46:11.684120Z","shell.execute_reply":"2022-12-21T06:46:12.650600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset=train_dataset, batch_size=250, shuffle=False)\n\nbatch_size_test = 250\ntest_loader = DataLoader(dataset=test_dataset, batch_size=batch_size_test, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:12.653244Z","iopub.execute_input":"2022-12-21T06:46:12.653601Z","iopub.status.idle":"2022-12-21T06:46:12.662517Z","shell.execute_reply.started":"2022-12-21T06:46:12.653565Z","shell.execute_reply":"2022-12-21T06:46:12.661460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch the iterator\ntest_iter = iter(test_loader)\n# Get the first batch\nimages, _ = test_iter.next()\n\n# Specify the figure width and height\nfig = plt.figure(figsize=(25, 4))\n\nfor i in range(1, 10):\n    ax = fig.add_subplot(2, 10, i, xticks=[], yticks=[])\n    plt.imshow(torch.moveaxis(images[i-1], 0, 2))","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:12.663862Z","iopub.execute_input":"2022-12-21T06:46:12.664602Z","iopub.status.idle":"2022-12-21T06:46:14.041884Z","shell.execute_reply.started":"2022-12-21T06:46:12.664555Z","shell.execute_reply":"2022-12-21T06:46:14.040899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:14.043371Z","iopub.execute_input":"2022-12-21T06:46:14.044010Z","iopub.status.idle":"2022-12-21T06:46:14.171847Z","shell.execute_reply.started":"2022-12-21T06:46:14.043971Z","shell.execute_reply":"2022-12-21T06:46:14.170678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if ALEXNET_TRAINING:\n        start_time = time.time()\n\n        epochs = 10\n        running_loss_history = []\n        running_corrects_history = []\n\n        for i in range(epochs):\n\n            running_loss = .0\n            running_corrects = .0\n\n            for batch_no, (X_train, y_train) in enumerate(train_loader):\n                # Increment the batch no.\n                batch_no += 1\n\n                X_train = X_train.to(device) # Move X_train to the GPU\n                y_train = y_train.to(device) # Move y_train to the GPU\n                gc.collect()\n                \n                y_pred = model_on_gpu.forward(X_train)\n\n                # Label data (y_train) needs to be flattened from [50,1] to [50] in order to calculate the loss\n                y_train = torch.flatten(y_train)\n\n                loss = criterion(y_pred, y_train.long()) # y_train has to be LONG\n\n                optimizer.zero_grad()\n                loss.backward()\n                optimizer.step()\n\n                running_loss += loss.item()\n\n                # Get the predicted numbers\n                _, y_pred = torch.max(y_pred, 1)\n\n                # Get the total number of the correct values\n                running_corrects += torch.sum(y_pred==y_train.data)\n            \n            # get the average loss and accuracy\n            epoch_loss = running_loss/50000 \n            running_loss_history.append(epoch_loss)\n            epoch_accuracy = running_corrects.float()/50000 \n            running_corrects_history.append(epoch_accuracy)\n\n            print(f'################## Epoch: {i} ##################')\n            print('Loss: {:.4f}, Accuracy: {:.4f}'.format(epoch_loss, epoch_accuracy))\n            print(f'Duration: {time.time() - start_time:.2f}')","metadata":{"execution":{"iopub.status.busy":"2022-12-21T06:46:14.173487Z","iopub.execute_input":"2022-12-21T06:46:14.173852Z","iopub.status.idle":"2022-12-21T07:12:29.715982Z","shell.execute_reply.started":"2022-12-21T06:46:14.173817Z","shell.execute_reply":"2022-12-21T07:12:29.714729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if ALEXNET_TRAINING:\n    plt.plot(running_loss_history, label='training_loss')","metadata":{"execution":{"iopub.status.busy":"2022-12-21T07:12:29.717846Z","iopub.execute_input":"2022-12-21T07:12:29.718545Z","iopub.status.idle":"2022-12-21T07:12:29.924439Z","shell.execute_reply.started":"2022-12-21T07:12:29.718507Z","shell.execute_reply":"2022-12-21T07:12:29.923642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### AlexNet Prediction and Submission","metadata":{}},{"cell_type":"code","source":"if ALEXNET_PREDICTION:\n    id_start = 0\n\n    for batch_no, (X_test, _) in enumerate(test_loader):\n\n        id_start = batch_no * batch_size_test + 1\n        id_end = id_start + batch_size_test - 1\n\n        # Increment the batch no.\n        batch_no += 1    \n\n        X_test = X_test.to(device) # Move X_train to the GPU\n\n        with torch.no_grad(): # Stop updating the gradient\n            _, y_test_pred = torch.max(model_on_gpu(X_test.float()), 1) # Pick the most probable label\n\n            # Move the predicted values to CPU. This may be heavy for the allocated RAM on the CPU.\n            y_test_pred = y_test_pred.cpu()   \n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # Convert it to numpy\n    #         y_test_pred = y_test_pred.numpy()\n\n            # Show the type for the confirmation purpose\n    #         print(type(y_test_pred))\n\n            # As submission expects actual category names such as cat, the predicted values have to be transformed. \n            y_test_pred = label_encoder.inverse_transform(y_test_pred)\n\n            # Reflect the predicted labels for the subjected range in submit_df\n            submit_df.loc[id_start-1:id_end-1, 'label'] = y_test_pred\n\n            if batch_no <= 1: # Print logs until the 2nd iteration\n                print(f' Batch #: {batch_no}')\n                print(f' Submission IDs: {id_start} - {id_end}')\n                print(submit_df[(submit_df.id >= id_start) & (submit_df.id <= id_end)])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T08:12:52.246200Z","iopub.execute_input":"2022-12-21T08:12:52.246930Z","iopub.status.idle":"2022-12-21T08:12:52.367149Z","shell.execute_reply.started":"2022-12-21T08:12:52.246831Z","shell.execute_reply":"2022-12-21T08:12:52.365573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if ALEXNET_PREDICTION:\n    # Check the result before the submission \n    display(submit_df.head())\n    submit_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-12-21T07:12:30.783624Z","iopub.status.idle":"2022-12-21T07:12:30.784302Z","shell.execute_reply.started":"2022-12-21T07:12:30.784006Z","shell.execute_reply":"2022-12-21T07:12:30.784034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch the iterator\ntest_iter = iter(test_loader)\n# Get the first batch\nimages, _ = test_iter.next()\n\n# Specify the figure width and height\nfig = plt.figure(figsize=(25, 4))\n\nfor i in range(1, 21):\n    ax = fig.add_subplot(2, 10, i, xticks=[], yticks=[])\n    plt.imshow(torch.moveaxis(images[i-1], 0, 2))\n    plt.title(submit_df['label'][i-1])","metadata":{"execution":{"iopub.status.busy":"2022-12-21T07:12:30.786275Z","iopub.status.idle":"2022-12-21T07:12:30.786761Z","shell.execute_reply.started":"2022-12-21T07:12:30.786517Z","shell.execute_reply":"2022-12-21T07:12:30.786540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This scored 0.8087, which tells how powerful AlexNet is compared to its predecessors.","metadata":{}},{"cell_type":"markdown","source":"### References\n1. [Dropout regularization technique](https://www.thepythoncode.com/article/dropout-regularization-in-pytorch)\n2. [Models supported by PyTorch](https://pytorch.org/vision/stable/models.html#classification)\n3. [Transfer Learning with Convolutional Neural Networks in PyTorch](https://towardsdatascience.com/transfer-learning-with-convolutional-neural-networks-in-pytorch-dd09190245ce)","metadata":{}},{"cell_type":"markdown","source":"Thank you!","metadata":{}}]}