{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30746,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Project setup","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries and modules\nimport torch\nimport torchvision\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport sklearn\nimport os\nimport seaborn as sns\nimport random\n\nfrom torch import nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import ConfusionMatrixDisplay\nfrom sklearn.metrics import accuracy_score\nfrom tqdm.auto import tqdm\nfrom typing import List, Dict, Tuple, Union\nfrom timeit import default_timer as timer\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\n# Try to get torchinfo, install it if it doesn't work\ntry:\n    from torchinfo import summary\nexcept:\n    print(\"[INFO] Couldn't find torchinfo. Installing.\")\n    !pip install -q torchinfo\n    from torchinfo import summary","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:01.133458Z","iopub.execute_input":"2024-07-27T23:37:01.133859Z","iopub.status.idle":"2024-07-27T23:37:06.868473Z","shell.execute_reply.started":"2024-07-27T23:37:01.133826Z","shell.execute_reply":"2024-07-27T23:37:06.867694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup device agnostic code\ndevice = \"cuda\" if torch.cuda.is_available() == True else \"cpu\"\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:06.870178Z","iopub.execute_input":"2024-07-27T23:37:06.871158Z","iopub.status.idle":"2024-07-27T23:37:06.907244Z","shell.execute_reply.started":"2024-07-27T23:37:06.871119Z","shell.execute_reply":"2024-07-27T23:37:06.906281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data preparation","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Load data","metadata":{}},{"cell_type":"code","source":"# Load the CSV data using pandas\ntrain_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:06.908412Z","iopub.execute_input":"2024-07-27T23:37:06.908747Z","iopub.status.idle":"2024-07-27T23:37:12.306800Z","shell.execute_reply.started":"2024-07-27T23:37:06.908722Z","shell.execute_reply":"2024-07-27T23:37:12.305942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.309031Z","iopub.execute_input":"2024-07-27T23:37:12.309331Z","iopub.status.idle":"2024-07-27T23:37:12.333324Z","shell.execute_reply.started":"2024-07-27T23:37:12.309307Z","shell.execute_reply":"2024-07-27T23:37:12.332461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.334446Z","iopub.execute_input":"2024-07-27T23:37:12.334749Z","iopub.status.idle":"2024-07-27T23:37:12.350412Z","shell.execute_reply.started":"2024-07-27T23:37:12.334725Z","shell.execute_reply":"2024-07-27T23:37:12.349565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Break up the training DataFrame into features and labels\nX_train = train_df.drop(labels=[\"label\"], axis=1)\ny_train = train_df[\"label\"]\n\n# Check number of entries or samples for each class\ny_train.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.351847Z","iopub.execute_input":"2024-07-27T23:37:12.352256Z","iopub.status.idle":"2024-07-27T23:37:12.450541Z","shell.execute_reply.started":"2024-07-27T23:37:12.352224Z","shell.execute_reply":"2024-07-27T23:37:12.449589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check number of entries or samples for each class with a graph using seaborn\nsns.countplot(x=train_df[\"label\"])\nplt.title(\"Class Distribution in Training Set\");","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.451759Z","iopub.execute_input":"2024-07-27T23:37:12.452042Z","iopub.status.idle":"2024-07-27T23:37:12.773429Z","shell.execute_reply.started":"2024-07-27T23:37:12.452017Z","shell.execute_reply":"2024-07-27T23:37:12.772472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.2 Check for `Null` and `NaN` values","metadata":{}},{"cell_type":"code","source":"# Check the data\nprint(train_df.isna().sum().any())\nprint(train_df.isnull().sum().any())\nprint(test_df.isna().sum().any())\nprint(test_df.isnull().sum().any())","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.774878Z","iopub.execute_input":"2024-07-27T23:37:12.775485Z","iopub.status.idle":"2024-07-27T23:37:12.858037Z","shell.execute_reply.started":"2024-07-27T23:37:12.775448Z","shell.execute_reply":"2024-07-27T23:37:12.857138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are no `Null` and `NaN` (missing) values. Therefore, we can safely proceed.","metadata":{}},{"cell_type":"markdown","source":"## 2.3 Partitioning training data","metadata":{}},{"cell_type":"code","source":"# Split up our training DataFrame inorder to create the validation DataFrame to validate our model\nX_train, X_val, y_train, y_val = train_test_split(X_train,\n                                                  y_train,\n                                                  test_size=0.2,\n                                                  random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:12.859477Z","iopub.execute_input":"2024-07-27T23:37:12.859802Z","iopub.status.idle":"2024-07-27T23:37:13.079007Z","shell.execute_reply.started":"2024-07-27T23:37:12.859777Z","shell.execute_reply":"2024-07-27T23:37:13.077954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:13.083244Z","iopub.execute_input":"2024-07-27T23:37:13.083890Z","iopub.status.idle":"2024-07-27T23:37:13.106128Z","shell.execute_reply.started":"2024-07-27T23:37:13.083859Z","shell.execute_reply":"2024-07-27T23:37:13.105128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:13.107542Z","iopub.execute_input":"2024-07-27T23:37:13.107856Z","iopub.status.idle":"2024-07-27T23:37:13.115082Z","shell.execute_reply.started":"2024-07-27T23:37:13.107832Z","shell.execute_reply":"2024-07-27T23:37:13.114109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:13.116180Z","iopub.execute_input":"2024-07-27T23:37:13.116452Z","iopub.status.idle":"2024-07-27T23:37:13.136296Z","shell.execute_reply.started":"2024-07-27T23:37:13.116428Z","shell.execute_reply":"2024-07-27T23:37:13.135268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.4 Visualize data\n\nDisplaying a few sample images from the dataset helps in getting to know the data better. It helps in EDA (explatory data analysis).","metadata":{}},{"cell_type":"code","source":"def plot_samples(X, y, num_samples: int=10):\n    # Adjust number of samples if it is too high\n    if num_samples > 10:\n        num_samples = 10\n        print(\"For display purposes, num_samples shouldn't be larger than 10, setting to 10.\")\n        pass\n    plt.figure(figsize=(10, 5))\n    \n    # Displaying the images\n    for i in range(num_samples):\n        plt.subplot(2, 5, i+1)\n        plt.imshow(X.iloc[i].values.reshape(28, 28), cmap=\"gray\")\n        plt.title(f\"Label: {y.iloc[i]}\")\n        plt.axis(False)\n    plt.tight_layout()\n    plt.show()\n    \n# Plot samples from training data\nplot_samples(X_train, y_train, num_samples=15)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:13.137445Z","iopub.execute_input":"2024-07-27T23:37:13.137761Z","iopub.status.idle":"2024-07-27T23:37:13.869480Z","shell.execute_reply.started":"2024-07-27T23:37:13.137735Z","shell.execute_reply":"2024-07-27T23:37:13.868584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check class distribution of training set after splitting of data\nsns.countplot(x=y_train)\nplt.title(\"Class Distribution in Training Set After Split\");","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:13.870817Z","iopub.execute_input":"2024-07-27T23:37:13.871184Z","iopub.status.idle":"2024-07-27T23:37:14.119402Z","shell.execute_reply.started":"2024-07-27T23:37:13.871150Z","shell.execute_reply":"2024-07-27T23:37:14.118459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check class distribution of validation set after splitting of data\nsns.countplot(x=y_val)\nplt.title(\"Class Distribution in Validation Set After Split\");","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.120415Z","iopub.execute_input":"2024-07-27T23:37:14.120693Z","iopub.status.idle":"2024-07-27T23:37:14.408323Z","shell.execute_reply.started":"2024-07-27T23:37:14.120669Z","shell.execute_reply":"2024-07-27T23:37:14.407276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can tell that our data has been split properly, so we are ready to proceed.","metadata":{}},{"cell_type":"markdown","source":"## 2.5 Concatenate validation and training separately","metadata":{}},{"cell_type":"code","source":"# Reset the indices of X_val and y_val to ensure they align properly\nX_val = X_val.reset_index(drop=True)\ny_val = y_val.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.409422Z","iopub.execute_input":"2024-07-27T23:37:14.409720Z","iopub.status.idle":"2024-07-27T23:37:14.434142Z","shell.execute_reply.started":"2024-07-27T23:37:14.409694Z","shell.execute_reply":"2024-07-27T23:37:14.433379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `reset_index(drop=True)` method ensures that the indices of `y_val` and `X_val` are reset so they match the indices of eachother to avoid any chances of having NaN values. If their will be NaN values, they will cause a huge problem later on when we train our model.","metadata":{}},{"cell_type":"code","source":"# Join X_val and y_val back into a singe DataFrame to create our validation DataFrame\nval_df = pd.concat([X_val, y_val], axis=1)\nval_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.435332Z","iopub.execute_input":"2024-07-27T23:37:14.435620Z","iopub.status.idle":"2024-07-27T23:37:14.476807Z","shell.execute_reply.started":"2024-07-27T23:37:14.435596Z","shell.execute_reply":"2024-07-27T23:37:14.475579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This concatenates `X_val` and `y_val` along the columns to form a single DataFrame.","metadata":{}},{"cell_type":"code","source":"# Ensure the label column is the first column\nval_df = val_df[[\"label\"] + [col for col in val_df.columns if col != \"label\"]]\nval_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.478162Z","iopub.execute_input":"2024-07-27T23:37:14.478688Z","iopub.status.idle":"2024-07-27T23:37:14.518132Z","shell.execute_reply.started":"2024-07-27T23:37:14.478650Z","shell.execute_reply":"2024-07-27T23:37:14.517308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reset the indices of X_val and y_val to ensure they align properly\nX_train = X_train.reset_index(drop=True)\ny_train = y_train.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.519199Z","iopub.execute_input":"2024-07-27T23:37:14.519483Z","iopub.status.idle":"2024-07-27T23:37:14.600511Z","shell.execute_reply.started":"2024-07-27T23:37:14.519459Z","shell.execute_reply":"2024-07-27T23:37:14.599695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `reset_index(drop=True)` method ensures that the indices of `y_train` and `X_train` are reset so they match the indices of eachother to avoid any chances of having NaN values. If their will be NaN values, they will cause a huge problem later on when we train our model.","metadata":{}},{"cell_type":"code","source":"# Join X_train and y_train back into a single DataFrame to create our training DataFrame\ntrain_df = pd.concat([X_train, y_train], axis=1)\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.601632Z","iopub.execute_input":"2024-07-27T23:37:14.601906Z","iopub.status.idle":"2024-07-27T23:37:14.705056Z","shell.execute_reply.started":"2024-07-27T23:37:14.601882Z","shell.execute_reply":"2024-07-27T23:37:14.704102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This concatenates `X_train` and `y_train` along the columns to form a single DataFrame.","metadata":{}},{"cell_type":"code","source":"# Ensure the label column is the first column\ntrain_df = train_df[[\"label\"] + [col for col in train_df.columns if col != \"label\"]]\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.708318Z","iopub.execute_input":"2024-07-27T23:37:14.708611Z","iopub.status.idle":"2024-07-27T23:37:14.796432Z","shell.execute_reply.started":"2024-07-27T23:37:14.708589Z","shell.execute_reply":"2024-07-27T23:37:14.795438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the data for Null and NaN values\nprint(train_df.isna().sum().any())\nprint(train_df.isnull().sum().any())\nprint(val_df.isna().sum().any())\nprint(val_df.isnull().sum().any())","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:37:14.797805Z","iopub.execute_input":"2024-07-27T23:37:14.798163Z","iopub.status.idle":"2024-07-27T23:37:14.851714Z","shell.execute_reply.started":"2024-07-27T23:37:14.798130Z","shell.execute_reply":"2024-07-27T23:37:14.850779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Create Datasets and DataLoaders","metadata":{}},{"cell_type":"markdown","source":"Normalization is a key concept in data preprocessing steps and is a very important concept to understand. \n\nFurthermore, normalization is just a preprocessing technique used to adjust the scale of the data to make it more suitable for training machine/deep learning models.\n\nYou may ask, why do we perform normalization?\n\n* Convergence Speed: Speeds up neural network training by stabilizing weights and gradients.\n* Illumination: Reduces sensitivity to varying lighting (how light or dark an image appears) conditions, focusing on patterns rather than absolute intensities.\n* Consistency: Ensures all input features are on the same scale, improving data consistency.\n\nIn summary, normalizing to [0, 1] makes training easier, reduces the impact of lighting changes, and ensures consistent input values.","metadata":{}},{"cell_type":"markdown","source":"## 3.1 Create and instantiate custom Dataset class","metadata":{}},{"cell_type":"markdown","source":"Remember, we have to create a custom `Dataset` class as PyTorch does not have a prebuilt custom class for this type of data. Also, we must redefine the `__getitem__` function as we are subclassing the `Dataset` class. Since we will be using `TrivialAugmentWide()`, we must transform our image into a PIL Image beforehand in order to use it, as its required. ","metadata":{}},{"cell_type":"code","source":"class DigitDataset(Dataset):\n    def __init__(self, dataframe: pd.DataFrame, train: bool = True):\n        \"\"\"\n        Initialize the dataset.\n\n        Parameters:\n        - dataframe: A pandas DataFrame containing the data.\n        - train: Whether the dataset is for training (includes labels) or testing (no labels).\n        \"\"\"\n        self.data = dataframe.values\n        self.train = train\n        self.num_columns = len(dataframe.columns)\n        \n        # Define transformations\n        if self.train == True:\n            self.transform = transforms.Compose([\n                transforms.ToPILImage(), # Convert NumPy array to PIL Image\n                transforms.TrivialAugmentWide(), # Apply a wide variety of data augmentations\n                transforms.ToTensor(), # Convert PIL Image to tensor and scale to [0, 1]\n                transforms.Normalize(mean=(0.5,), std=(0.5,)) # Normalize the tensor\n            ])\n        else:\n            self.transform = transforms.Compose([\n                transforms.ToTensor(), # Convert NumPy array to tensor and scale to [0, 1]\n                transforms.Normalize(mean=(0.5,), std=(0.5,)) # Normalize the tensor\n            ])\n            \n    def __len__(self) -> int:\n        \"\"\"Return the number of samples in the dataset.\"\"\"\n        return len(self.data)\n    \n    def __getitem__(self, idx: int) -> Tuple[torch.Tensor, Union[torch.Tensor, None]]:\n        \"\"\"\n        Retrieve a sample from the dataset at the given index.\n\n        Parameters:\n        - idx: The index of the sample to retrieve.\n\n        Returns:\n        - A tuple containing the sample (features and label) for training, or just features for testing.\n        \"\"\"\n        data_sample = self.data[idx]\n        \n        if self.num_columns == 785:\n            features = data_sample[1:].reshape(28, 28, 1) # For training and validation DataFrames as they have a label column\n        else:\n            features = data_sample.reshape(28, 28, 1)  # For testing DataFrame as it doesn't have a label column\n        \n        # Apply transformations\n        features = self.transform(features.astype(np.uint8))  # Convert to uint8 for PIL Image compatibility\n        \n        if self.num_columns == 785:\n            label = int(data_sample[0])\n            return features, torch.tensor(label, dtype=torch.long)\n        else:\n            return features\n        ","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:56:41.729638Z","iopub.execute_input":"2024-07-27T23:56:41.730008Z","iopub.status.idle":"2024-07-27T23:56:41.741488Z","shell.execute_reply.started":"2024-07-27T23:56:41.729976Z","shell.execute_reply":"2024-07-27T23:56:41.740578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `transforms.ToPILImage()` function from `torchvision.transforms` is used to convert tensors or NumPy arrays into PIL Images. It expects input data in tensor format with shape `(C, H, W)` (for tensors) or `(H, W)`/`(H, W, C)` (for NumPy arrays), where `C` denotes the number of channels, and `H` and `W` represent the height and width of the image. Pixel values should generally be in the range `[0, 1]` for floating-point tensors or `[0, 255]` for integer types.","metadata":{}},{"cell_type":"code","source":"# Create an instance of the training, validation, and test datasets\ntrain_data = DigitDataset(dataframe=train_df, train=True)\nval_data = DigitDataset(dataframe=val_df, train=False)\ntest_data = DigitDataset(dataframe=test_df, train=False)\n\ntrain_data, val_data, test_data","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:56:42.360400Z","iopub.execute_input":"2024-07-27T23:56:42.361467Z","iopub.status.idle":"2024-07-27T23:56:42.463363Z","shell.execute_reply.started":"2024-07-27T23:56:42.361416Z","shell.execute_reply":"2024-07-27T23:56:42.462502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 Identifying and storing image classes","metadata":{}},{"cell_type":"markdown","source":"The `.append()` and `.extend()` methods are used to modify lists in Python. `.append()` adds a single element to the end of a list, regardless of its type, which may include another list. For example, `my_list.append([4, 5])` adds `[4, 5]` as a single element. In contrast, `.extend()` iterates over an iterable (like another list or tuple) and adds each of its elements to the list. For instance, `my_list.extend([4, 5])` adds each element of `[4, 5]` individually.","metadata":{}},{"cell_type":"code","source":"# Initialize a list to store labels\nall_labels = []\n\n# Iterate through the dataset to collect all labels\nfor image, label in train_data:\n     all_labels.append(label.tolist())\n        \n# Convert the list to a unique list of class names\nclass_names = list(set(all_labels))\nclass_names","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:42:27.421474Z","iopub.execute_input":"2024-07-27T23:42:27.422149Z","iopub.status.idle":"2024-07-27T23:42:44.410444Z","shell.execute_reply.started":"2024-07-27T23:42:27.422118Z","shell.execute_reply":"2024-07-27T23:42:44.409560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.3 Display random images","metadata":{}},{"cell_type":"code","source":"# Take in a Dataset as well as a list of class names\ndef display_random_images(dataset: torch.utils.data.dataset.Dataset,\n                          classes: List[str] = None,\n                          n: int = 10,\n                          display_shape: bool = True,\n                          seed: int = None):\n    \n    # Adjust display and n if n is too high    \n    if n > 10:\n        n = 10\n        display_shape = False\n        print(\"For display purposes, n shouldn't be larger than 10, setting to 10 and removing shape display.\")\n        \n    # Set random seed if give\n    if seed:\n        random.seed(seed)\n        \n    # Get random sample indexes (random sampling)\n    random_samples_idx = random.sample(range(len(dataset)), k=n)\n    \n    # Setup plot\n    plt.figure(figsize=(16, 8))\n    \n    # Loop through the samples and display them randomly\n    for i, sample in enumerate(random_samples_idx):\n        image, label = dataset[sample][0], dataset[sample][1]\n        \n        # Adjust image tensor shape for plotting: [color_channels, height, width] -> [height, width, color_channels]\n        image = image.permute(1, 2, 0)\n        \n        # Plot adjusted samples\n        plt.subplot(2, 5, i+1)\n        plt.imshow(image, cmap=\"gray\")\n        plt.axis(False)\n        title = \"\"\n        if classes:\n            title = f\"Class: {classes[label]}\"\n            if display_shape:\n                title = title + f\"\\nShape: {image.shape}\"\n        plt.title(title);\n        \n# Display random images from custom DigitDataset\ndisplay_random_images(train_data,\n                      n=10,\n                      classes=class_names)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:42:44.411895Z","iopub.execute_input":"2024-07-27T23:42:44.412150Z","iopub.status.idle":"2024-07-27T23:42:45.303416Z","shell.execute_reply.started":"2024-07-27T23:42:44.412129Z","shell.execute_reply":"2024-07-27T23:42:45.302506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NHWC**: (Used by TensorFlow, Matplotlib, NumPy)\n\nN: Number of samples. H: Height of the input data. W: Width of the input data. C: Number of channels (e.g., RGB for images)\n\n**NCHW**: (Used by PyTorch)\n\nN: Number of samples. C: Number of channels (e.g., RGB for images) H: Height of the input data. W: Width of the input data.","metadata":{}},{"cell_type":"markdown","source":"## 3.4 Turn custom loaded data into DataLoaders ","metadata":{}},{"cell_type":"markdown","source":"Setting `pin_memory=True` in a PyTorch `DataLoader` helps speed up the transfer of data from your computer's memory to the GPU. This makes training faster and smoother when using a GPU. It's usually a good idea to enable this if you're using a GPU.","metadata":{}},{"cell_type":"code","source":"# Setup hyperparameters\nBATCH_SIZE = 100\nNUM_WORKERS = os.cpu_count()\n\n# Turn custom/loaded data/images into DataLoaders\ntrain_dataloader = DataLoader(dataset=train_data,\n                              batch_size=BATCH_SIZE,\n                              num_workers=NUM_WORKERS,\n                              shuffle=True,\n                              pin_memory=True)\nval_dataloader = DataLoader(dataset=val_data,\n                            batch_size=BATCH_SIZE,\n                            num_workers=NUM_WORKERS,\n                            shuffle=False,\n                            pin_memory=True)\ntest_dataloader = DataLoader(dataset=test_data,\n                             batch_size=BATCH_SIZE,\n                             num_workers=NUM_WORKERS,\n                             shuffle=False,\n                             pin_memory=True)\n\ntrain_dataloader, val_dataloader, test_dataloader","metadata":{"execution":{"iopub.status.busy":"2024-07-28T00:03:25.841244Z","iopub.execute_input":"2024-07-28T00:03:25.841749Z","iopub.status.idle":"2024-07-28T00:03:25.850762Z","shell.execute_reply.started":"2024-07-28T00:03:25.841716Z","shell.execute_reply":"2024-07-28T00:03:25.849913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Create model with data augmentation : `TinyVGG`","metadata":{}},{"cell_type":"markdown","source":"Now, we will create our model with augmented data. Our data already has been augmented in our custom `Dataset` class. Therefore, we don't have to do it again.\n\nYou may ask, what is data augmentation?\n\nWell, data augmentation is a technique used to artificially increase the size of a dataset by creating modified versions of existing data. This includes transformations like rotating, flipping, or changing the color of images. It helps improve a model's performance by making it more robust and better at handling variations in real-world data. \n\nThe data augmentation part in our code is the `transforms.TrivialAugmentWide()` in the `self.transforms` in the custom `DigitDataset` class.","metadata":{}},{"cell_type":"markdown","source":"## 4.1 Create TinyVGG model class\n\nFor this project for recognizing digits, I will be using the TinyVGG architecture. Learn more about it here - https://poloclub.github.io/cnn-explainer/. Except, I would be increasing the number of hidden units to improve the model as 10 hidden units are not sufficient.","metadata":{}},{"cell_type":"code","source":"class TinyVGG(nn.Module):\n    \"\"\"\n    Model architecture copying TinyVGG from: \n    https://poloclub.github.io/cnn-explainer/\n    \"\"\"\n    def __init__(self,\n                 input_shape: int,\n                 hidden_units: int,\n                 output_shape: int):\n        super().__init__()\n        self.conv_block_1 = nn.Sequential(\n            nn.Conv2d(in_channels=input_shape,\n                      out_channels=hidden_units,\n                      kernel_size=(3, 3),\n                      stride=1,\n                      padding=1),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=hidden_units,\n                      out_channels=hidden_units,\n                      kernel_size=(3, 3),\n                      stride=1,\n                      padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(2, 2))\n        )\n        self.conv_block_2 = nn.Sequential(\n            nn.Conv2d(in_channels=hidden_units,\n                      out_channels=hidden_units,\n                      kernel_size=(3, 3),\n                      stride=1,\n                      padding=1),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=hidden_units,\n                      out_channels=hidden_units,\n                      kernel_size=(3, 3),\n                      stride=1,\n                      padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(2, 2))\n        )\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(in_features=hidden_units*7*7,\n                      out_features=output_shape)\n        )\n        \n    def forward(self,\n                x: torch.Tensor()) -> torch.Tensor():\n        return self.classifier(self.conv_block_2(self.conv_block_1(x))) # Leverage benefits of operator fusion\n    \nmodel_0 = TinyVGG(input_shape=1,\n                  hidden_units=64,\n                  output_shape=len(class_names)).to(device)\nmodel_0","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:11.824861Z","iopub.execute_input":"2024-07-27T23:43:11.825223Z","iopub.status.idle":"2024-07-27T23:43:11.985351Z","shell.execute_reply.started":"2024-07-27T23:43:11.825194Z","shell.execute_reply":"2024-07-27T23:43:11.984384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.2 Try a forward pass on a single image","metadata":{}},{"cell_type":"markdown","source":"This allows us to test our model and see if it is working. It helps us out if there are any shape errors like the one I had for the `in_features` of the `nn.Linear` layer in the `self.classifier` block.","metadata":{}},{"cell_type":"code","source":"# 1. Get a batch of images and labels from the DataLoader\nimage_batch, label_batch = next(iter(train_dataloader))\n\n# 2. Get a single image from the batch and unsqueeze the image so its shape fits the model\nimage_single, label_single = image_batch[0].unsqueeze(dim=0).to(device), label_batch[0].to(device)\nprint(f\"Single Image Shape:\\n{image_single.shape}\\n\")\n\n# Make a prediction on that single image\nmodel_0.eval()\nwith torch.inference_mode():\n    pred = model_0(image_single)\n    pred_prob = pred.softmax(dim=1)\n    pred_label = pred_prob.argmax(dim=1)\n    \n# Print out what's happening throughout the prediction process\nprint(f\"Output Logit:\\n{pred}\\n\")\nprint(f\"Output Prediction Probabilitie:\\n{pred_prob}\\n\")\nprint(f\"Output Prediction Label:\\n{pred_label}\\n\")\nprint(f\"Actual Label:\\n{label_single}\")","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:12.402339Z","iopub.execute_input":"2024-07-27T23:43:12.403189Z","iopub.status.idle":"2024-07-27T23:43:13.423557Z","shell.execute_reply.started":"2024-07-27T23:43:12.403155Z","shell.execute_reply":"2024-07-27T23:43:13.422444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.3 Getting a summary of our model with `torchinfo.summary()`","metadata":{}},{"cell_type":"code","source":"# Print a summary using torchinfo\nsummary(model=model_0,\n        input_size=(1, 1, 28, 28),\n        col_names=[\"input_size\", \"output_size\", \"num_params\", \"trainable\"],\n        col_width=20,\n        row_settings=[\"var_names\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:13.425786Z","iopub.execute_input":"2024-07-27T23:43:13.426569Z","iopub.status.idle":"2024-07-27T23:43:13.440373Z","shell.execute_reply.started":"2024-07-27T23:43:13.426505Z","shell.execute_reply":"2024-07-27T23:43:13.439338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Train `TinyVGG` model","metadata":{}},{"cell_type":"code","source":"# Setup loss function and optimizer\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.RMSprop(params=model_0.parameters(),\n                                lr=0.001)\n\n# Initialize the learning rate scheduler\nscheduler = ReduceLROnPlateau(optimizer,\n                              mode=\"max\",\n                              factor=0.5,\n                              patience=3,\n                              min_lr=0.00001,\n                              verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:13.441603Z","iopub.execute_input":"2024-07-27T23:43:13.442149Z","iopub.status.idle":"2024-07-27T23:43:13.448121Z","shell.execute_reply.started":"2024-07-27T23:43:13.442117Z","shell.execute_reply":"2024-07-27T23:43:13.447214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`ReduceLROnPlateau()` adjusts the learning rate when a metric has stopped improving. In this case, the metric is the validation accuracy or `val_acc`.\n\nHere’s a breakdown of the parameters:\n- `monitor='val_acc'`: Monitors the validation accuracy. When this metric stops improving, the learning rate will be reduced.\n- `patience=3`: Number of epochs to wait for improvement before reducing the learning rate.\n- `verbose=1`: Prints a message when the learning rate is reduced.\n- `factor=0.5`: The factor by which the learning rate will be reduced (e.g., if the learning rate was 0.01, it will be reduced to 0.005).\n- `min_lr=0.00001`: The minimum learning rate to which it can be reduced.\n- `mode='max'`: Specifies whether higher values of the metric are better (e.g., accuracy), or 'min' if monitoring a metric where lower values are preferable (e.g., loss).","metadata":{}},{"cell_type":"code","source":"def train_step(model: torch.nn.Module,\n               train_dataloader: torch.utils.data,\n               loss_fn: torch.nn.Module,\n               optimizer: torch.optim,\n               accuracy_score: sklearn.metrics = accuracy_score,\n               device: torch.device = device):\n    # Put the model in training mode\n    model.train()\n\n    # Setup train loss and train accuracy\n    train_loss, train_acc = 0, 0\n\n    # Loop through data loader data batches\n    for batch, (X_train, y_train) in enumerate(train_dataloader):\n        # Send the data to the target device\n        X_train, y_train = X_train.to(device), y_train.to(device)\n\n        # 1. Forward propagation\n        y_logits = model(X_train)\n        y_preds = y_logits.argmax(dim=1)\n\n        # 2. Calculate the loss and accuracy\n        loss = loss_fn(y_logits, y_train)\n        train_loss += loss.item()\n        acc = accuracy_score(y_train.cpu().numpy(), y_preds.cpu().numpy())*100\n        train_acc += acc\n\n        # 3. Optimizer zero grad\n        optimizer.zero_grad()\n\n        # 4. Backpropagation\n        loss.backward()\n\n        # 5. Optimizer step (gradient descent)\n        optimizer.step()\n\n    # Calculate the loss and accuracy per epoch\n    train_loss /= len(train_dataloader)\n    train_acc /= len(train_dataloader)\n    return train_loss, train_acc","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:13.578906Z","iopub.execute_input":"2024-07-27T23:43:13.579462Z","iopub.status.idle":"2024-07-27T23:43:13.587351Z","shell.execute_reply.started":"2024-07-27T23:43:13.579438Z","shell.execute_reply":"2024-07-27T23:43:13.586476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_step(model: torch.nn.Module,\n             val_dataloader: torch.utils.data,\n             loss_fn: torch.nn.Module,\n             accuracy_score: sklearn.metrics = accuracy_score,\n             device: torch.device = device):\n    # Put the model in evaluation mode\n    model.eval()\n\n    # Setup val loss and val accuracy\n    val_loss, val_acc = 0, 0\n\n    # Turn on inference context manager\n    with torch.inference_mode():\n        # Loop through data loader data batches\n        for (X_val, y_val) in val_dataloader:\n            # Send the data to the target device\n            X_val, y_val = X_val.to(device), y_val.to(device)\n\n            # 1. Forward propagation\n            val_logits = model(X_val)\n            val_preds = val_logits.argmax(dim=1)\n\n            # 2. Calculate the loss and accuracy\n            val_loss += loss_fn(val_logits, y_val).item()\n            val_acc += accuracy_score(y_val.cpu().numpy(), val_preds.cpu().numpy())*100\n\n        # Calculate the loss and accuracy per epoch\n        val_loss /= len(val_dataloader)\n        val_acc /= len(val_dataloader)\n        return val_loss, val_acc","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:13.780208Z","iopub.execute_input":"2024-07-27T23:43:13.780505Z","iopub.status.idle":"2024-07-27T23:43:13.788134Z","shell.execute_reply.started":"2024-07-27T23:43:13.780481Z","shell.execute_reply":"2024-07-27T23:43:13.787134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import tqdm for a progress bar\nfrom tqdm.auto import tqdm\n\ndef train(model: torch.nn.Module,\n          train_dataloader: torch.utils.data,\n          val_dataloader: torch.utils.data,\n          loss_fn: torch.nn.Module,\n          optimizer: torch.optim,\n          epochs: int,\n          accuracy_score: sklearn.metrics = accuracy_score,\n          device: torch.device = device):\n    # Create empty results dictionary\n    results = {\"train_loss\": [],\n               \"train_acc\": [],\n               \"val_loss\": [],\n               \"val_acc\": []}\n\n    # Loop through training and validation steps for a number of epochs\n    for epoch in tqdm(range(epochs)):\n        train_loss, train_acc = train_step(model=model,\n                                           train_dataloader=train_dataloader,\n                                           loss_fn=loss_fn,\n                                           optimizer=optimizer,\n                                           accuracy_score=accuracy_score,\n                                           device=device)\n        val_loss, val_acc = val_step(model=model,\n                                     val_dataloader=val_dataloader,\n                                     loss_fn=loss_fn,\n                                     accuracy_score=accuracy_score,\n                                     device=device)\n\n        # Print out what's happening\n        print(f\"Epoch: {epoch+1}\\n--------------------\")\n        print(f\"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.2f}% | Val Loss: {val_loss:.4f} | Val Acc: {val_acc:.2f}%\\n\")\n\n        # Update results dictionary\n        results[\"train_loss\"].append(train_loss)\n        results[\"train_acc\"].append(train_acc)\n        results[\"val_loss\"].append(val_loss)\n        results[\"val_acc\"].append(val_acc)\n        \n        # Update the scheduler with the validation accuracy (or loss if you wanted to)\n        scheduler.step(val_acc)\n\n    # Return the filled results at the end of the epochs\n    return results","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:13.976337Z","iopub.execute_input":"2024-07-27T23:43:13.976717Z","iopub.status.idle":"2024-07-27T23:43:13.986328Z","shell.execute_reply.started":"2024-07-27T23:43:13.976685Z","shell.execute_reply":"2024-07-27T23:43:13.985338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the random seeds\ntorch.manual_seed(42)\ntorch.cuda.manual_seed(42)\n\n# Set number of epochs training will run for\nNUM_EPOCHS = 20\n\n# Start measuring the time to complete training of model 0\nfrom timeit import default_timer as timer\nstart_time = timer()\n\n# Train model 0\nmodel_0_results = train(model=model_0,\n                        train_dataloader=train_dataloader,\n                        val_dataloader=val_dataloader,\n                        loss_fn=loss_fn,\n                        optimizer=optimizer,\n                        epochs=NUM_EPOCHS,\n                        accuracy_score=accuracy_score,\n                        device=device)\n\n# End measuring the time to complete training of model 0 and print out how long it took\nend_time = timer()\ntotal_train_time_model_0 = end_time - start_time\nprint(f\"[INFO] Train time on {device}: {total_train_time_model_0:.3f} seconds.\")","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:43:14.165341Z","iopub.execute_input":"2024-07-27T23:43:14.165678Z","iopub.status.idle":"2024-07-27T23:46:12.873850Z","shell.execute_reply.started":"2024-07-27T23:43:14.165652Z","shell.execute_reply":"2024-07-27T23:46:12.872760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, after 20 epochs, there is no significant increase in the validation accuracy. Therefore, now we'll improve by using transfer learning to see if the model can get about the same accuracy in less time.","metadata":{}},{"cell_type":"markdown","source":"# 6. Getting a pretrained model : `ResNet18`\n\nFor this project of recognizing digits, I'll be using `ResNet18` as the pretrained model from the PyTorch torchvision domain library to see if we can improve the speed of our model. Learn more about it here - https://pytorch.org/vision/stable/models/generated/torchvision.models.resnet18.html#torchvision.models.ResNet18_Weights","metadata":{}},{"cell_type":"markdown","source":"## 6.1 Setting up pretrained `ResNet18` model","metadata":{}},{"cell_type":"code","source":"# Setup pretrained ResNet18 model and send it to the target device\nmodel_1 = torchvision.models.resnet18(weights=None, num_classes=len(class_names)).to(device)\n# model_1 # Uncomment to see model's architecture (it's pretty long)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:12.875806Z","iopub.execute_input":"2024-07-27T23:46:12.876108Z","iopub.status.idle":"2024-07-27T23:46:13.107108Z","shell.execute_reply.started":"2024-07-27T23:46:12.876077Z","shell.execute_reply":"2024-07-27T23:46:13.106335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The pretrained model we are using has an input size with 3 color channels. As our data is in grayscale, its color channels is 1. Due to this, we will change the first layer of the model which is a `nn.Conv2d` to expect 1 color channels instead of 3.","metadata":{}},{"cell_type":"code","source":"# Modify the first convolutional layer to accept 1 input/color channel\nmodel_1.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:13.108381Z","iopub.execute_input":"2024-07-27T23:46:13.108755Z","iopub.status.idle":"2024-07-27T23:46:13.114371Z","shell.execute_reply.started":"2024-07-27T23:46:13.108721Z","shell.execute_reply":"2024-07-27T23:46:13.113365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.2 Getting a summary of our model with `torchinfo.summary()`","metadata":{}},{"cell_type":"code","source":"# Get an image and label from the training dataloader and check the shape of them\nimage_batch, label_batch = next(iter(train_dataloader))\nimage_single, label_single = image_batch[0].unsqueeze(dim=0).to(device), label_batch[0].to(device)\nprint(image_single.shape, label_single.shape)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:13.116722Z","iopub.execute_input":"2024-07-27T23:46:13.117012Z","iopub.status.idle":"2024-07-27T23:46:13.386759Z","shell.execute_reply.started":"2024-07-27T23:46:13.116989Z","shell.execute_reply":"2024-07-27T23:46:13.385592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print a summary using torchinfo\nsummary(model=model_1,\n        input_size=(1, 1, 28, 28),\n        col_names=[\"input_size\", \"output_size\", \"num_params\", \"trainable\"],\n        col_width=20,\n        row_settings=[\"var_names\"])","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:13.388448Z","iopub.execute_input":"2024-07-27T23:46:13.388827Z","iopub.status.idle":"2024-07-27T23:46:13.479218Z","shell.execute_reply.started":"2024-07-27T23:46:13.388795Z","shell.execute_reply":"2024-07-27T23:46:13.478266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Train `ResNet18` model","metadata":{}},{"cell_type":"code","source":"# Setup loss function and optimizer\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(params=model_1.parameters(), \n                             lr=0.001)\n\n# Initialize the learning rate scheduler\nscheduler = ReduceLROnPlateau(optimizer,\n                              mode=\"max\",\n                              factor=0.5,\n                              patience=3,\n                              min_lr=0.00001,\n                              verbose=1)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:13.480577Z","iopub.execute_input":"2024-07-27T23:46:13.481193Z","iopub.status.idle":"2024-07-27T23:46:13.488075Z","shell.execute_reply.started":"2024-07-27T23:46:13.481159Z","shell.execute_reply":"2024-07-27T23:46:13.487058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the random seeds\ntorch.manual_seed(42)\ntorch.cuda.manual_seed(42)\n\n# Set number of epochs training will run for\nNUM_EPOCHS = 20\n\n# Start measuring the time to complete training of model 0\nfrom timeit import default_timer as timer\nstart_time = timer()\n\n# Train model 1\nmodel_1_results = train(model=model_1,\n                        train_dataloader=train_dataloader,\n                        val_dataloader=val_dataloader,\n                        loss_fn=loss_fn,\n                        optimizer=optimizer,\n                        epochs=NUM_EPOCHS,\n                        accuracy_score=accuracy_score,\n                        device=device)\n\n# End measuring the time to complete training of model 1 and print out how long it took\nend_time = timer()\ntotal_train_time_model_1 = end_time - start_time\nprint(f\"[INFO] Train time on {device}: {total_train_time_model_1:.3f} seconds.\")","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:46:13.489159Z","iopub.execute_input":"2024-07-27T23:46:13.489424Z","iopub.status.idle":"2024-07-27T23:49:38.464352Z","shell.execute_reply.started":"2024-07-27T23:46:13.489401Z","shell.execute_reply":"2024-07-27T23:49:38.463289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see, there isn't any crazy difference in performance between the models. Therefore, we will just use the `TinyVGG` model as the number of parameters and the size of it is relatively lower. Also, the `TinyVGG` model has a faster time.","metadata":{}},{"cell_type":"markdown","source":"# 8. Evaluate models\n\nWe've trained two different models.\n\n1. `model_0` - Our baseline model which mimics the TinyVGG architecture on the CNN Explainer website.\n2. `model_1` - Our pretrained model from `torchvision.models` section of the PyTorch library\n","metadata":{}},{"cell_type":"markdown","source":"## 8.1 Plot training and validation curves","metadata":{}},{"cell_type":"code","source":"# Convert our model results dictionaries into DataFrames\nmodel_0_df = pd.DataFrame(model_0_results)\nmodel_1_df = pd.DataFrame(model_1_results)","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:38.466315Z","iopub.execute_input":"2024-07-27T23:49:38.466798Z","iopub.status.idle":"2024-07-27T23:49:38.473752Z","shell.execute_reply.started":"2024-07-27T23:49:38.466756Z","shell.execute_reply":"2024-07-27T23:49:38.472866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_0_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:38.474798Z","iopub.execute_input":"2024-07-27T23:49:38.475050Z","iopub.status.idle":"2024-07-27T23:49:38.495663Z","shell.execute_reply.started":"2024-07-27T23:49:38.475028Z","shell.execute_reply":"2024-07-27T23:49:38.494853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1_df","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:38.498167Z","iopub.execute_input":"2024-07-27T23:49:38.498439Z","iopub.status.idle":"2024-07-27T23:49:38.516661Z","shell.execute_reply.started":"2024-07-27T23:49:38.498416Z","shell.execute_reply":"2024-07-27T23:49:38.515341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setup a plot \nplt.figure(figsize=(15, 10))\n\n# Get number of epochs\nepochs = range(len(model_0_df))\n\n# Plot train loss\nplt.subplot(2, 2, 1)\nplt.plot(epochs, model_0_df[\"train_loss\"], label=\"Model 0\")\nplt.plot(epochs, model_1_df[\"train_loss\"], label=\"Model 1\")\nplt.title(\"Train Loss\")\nplt.xlabel(\"Epochs\")\nplt.legend()\n\n# Plot test loss\nplt.subplot(2, 2, 2)\nplt.plot(epochs, model_0_df[\"val_loss\"], label=\"Model 0\")\nplt.plot(epochs, model_1_df[\"val_loss\"], label=\"Model 1\")\nplt.title(\"Test Loss\")\nplt.xlabel(\"Epochs\")\nplt.legend()\n\n# Plot train accuracy\nplt.subplot(2, 2, 3)\nplt.plot(epochs, model_0_df[\"train_acc\"], label=\"Model 0\")\nplt.plot(epochs, model_1_df[\"train_acc\"], label=\"Model 1\")\nplt.title(\"Train Accuracy\")\nplt.xlabel(\"Epochs\")\nplt.legend()\n\n# Plot test accuracy\nplt.subplot(2, 2, 4)\nplt.plot(epochs, model_0_df[\"val_acc\"], label=\"Model 0\")\nplt.plot(epochs, model_1_df[\"val_acc\"], label=\"Model 1\")\nplt.title(\"Test Accuracy\")\nplt.xlabel(\"Epochs\")\nplt.legend();","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:38.520251Z","iopub.execute_input":"2024-07-27T23:49:38.520893Z","iopub.status.idle":"2024-07-27T23:49:39.573979Z","shell.execute_reply.started":"2024-07-27T23:49:38.520860Z","shell.execute_reply":"2024-07-27T23:49:39.573016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.2 Plot total train times","metadata":{}},{"cell_type":"code","source":"# Setup a plot\nplt.figure(figsize=(5, 5))\n\n# Get the data to plot\nmodels = [\"Model 0 (TinyVGG)\", \"Model 1 (ResNet18)\"]\ntotal_train_times = [total_train_time_model_0, total_train_time_model_1]\n\n# Plot a bar plot to compare total training times\nplt.bar(models, total_train_times)\nplt.ylabel(\"Total Train Times (Sec)\");","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:39.575356Z","iopub.execute_input":"2024-07-27T23:49:39.575707Z","iopub.status.idle":"2024-07-27T23:49:39.793765Z","shell.execute_reply.started":"2024-07-27T23:49:39.575680Z","shell.execute_reply":"2024-07-27T23:49:39.792912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.3 Confusion matrix","metadata":{}},{"cell_type":"code","source":"# Make predictions with trained model\npreds = []\ntrue = []\nmodel_0.eval()\nwith torch.inference_mode():\n    for X, y in tqdm(val_dataloader, desc=\"Making Predictions\"):\n        # Send the data to the target device\n        X, y = X.to(device), y.to(device)\n        \n        # Forward propagation\n        logits = model_0(X)\n        labels = logits.argmax(dim=1)\n        \n        # Put predictions on CPU for evaluation and append them to predictions list\n        preds.append(labels.cpu())\n        \n        # Put truth values on CPU and append them to truth values list\n        true.append(y.cpu())\n        \n# Concatenate list of predictions into a tensor and truth labels into a tensor\npreds_tensor = torch.cat(preds)\ntrue_tensor = torch.cat(true)\npreds_tensor","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:39.794808Z","iopub.execute_input":"2024-07-27T23:49:39.795076Z","iopub.status.idle":"2024-07-27T23:49:40.631727Z","shell.execute_reply.started":"2024-07-27T23:49:39.795052Z","shell.execute_reply":"2024-07-27T23:49:40.630590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display confusion matrix\nConfusionMatrixDisplay.from_predictions(y_true=true_tensor,\n                                        y_pred=preds_tensor,\n                                        display_labels=class_names,\n                                        cmap=plt.cm.Blues,\n                                        colorbar=True);","metadata":{"execution":{"iopub.status.busy":"2024-07-27T23:49:40.633605Z","iopub.execute_input":"2024-07-27T23:49:40.633936Z","iopub.status.idle":"2024-07-27T23:50:01.206113Z","shell.execute_reply.started":"2024-07-27T23:49:40.633902Z","shell.execute_reply":"2024-07-27T23:50:01.205212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9. Submit to Kaggle in proper format","metadata":{}},{"cell_type":"code","source":"# Iterate over the test dataloader making predictions for each batch\nmodel_0.eval()\npreds = []\nwith torch.inference_mode():\n    for X_test in test_dataloader:\n        X_test = X_test.to(device)\n        logits = model_0(X_test)\n        labels = logits.argmax(dim=1)\n        preds.extend(labels.cpu().numpy())","metadata":{"execution":{"iopub.status.busy":"2024-07-28T00:03:35.497077Z","iopub.execute_input":"2024-07-28T00:03:35.497716Z","iopub.status.idle":"2024-07-28T00:03:37.118591Z","shell.execute_reply.started":"2024-07-28T00:03:35.497683Z","shell.execute_reply":"2024-07-28T00:03:37.117548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a pandas DataFrame with the required columns for submission\nsubmission = pd.DataFrame({\"ImageId\": np.arange(1, len(preds) + 1), \"Label\": preds})\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2024-07-28T00:05:57.678280Z","iopub.execute_input":"2024-07-28T00:05:57.678772Z","iopub.status.idle":"2024-07-28T00:05:57.778384Z","shell.execute_reply.started":"2024-07-28T00:05:57.678740Z","shell.execute_reply":"2024-07-28T00:05:57.777286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the DataFrame as a CSV file\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2024-07-28T00:07:00.253031Z","iopub.execute_input":"2024-07-28T00:07:00.253413Z","iopub.status.idle":"2024-07-28T00:07:00.293001Z","shell.execute_reply.started":"2024-07-28T00:07:00.253381Z","shell.execute_reply":"2024-07-28T00:07:00.292175Z"},"trusted":true},"execution_count":null,"outputs":[]}]}