{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CIFAR-10 Image Classification - PyTorch 🔥 CNN","metadata":{"execution":{"iopub.status.busy":"2023-09-19T17:23:29.286279Z","iopub.execute_input":"2023-09-19T17:23:29.286689Z","iopub.status.idle":"2023-09-19T17:23:29.293317Z","shell.execute_reply.started":"2023-09-19T17:23:29.286653Z","shell.execute_reply":"2023-09-19T17:23:29.291838Z"}}},{"cell_type":"markdown","source":"Goal: Building a CNN Classificator with PyTorch to classify plant seedlings from differently sized images.\n\nData: Plant Seedlings Classification via Kaggle (https://www.kaggle.com/competitions/plant-seedlings-classification)\n\nProcedure:\n\nPreviewing images\n- Unzip dataset takes 95% of the Kernel's runtime...) \n- Load Images\n- Normalize\n- Encode Labels\n- Train/Validation Split\n- Create DataLoader\n- Create CNN Model with pytorch\n- Train Model\n- Evaluate Loss, micro-F1, and Accuracy over Time\n- Predict Labels for Test Data and create submission file\n\nOthers:\n- Compatible with Google Colab and Kaggle as runtime\n- CUDA support","metadata":{}},{"cell_type":"code","source":"import os\nimport torch\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f'Running on {DEVICE}')\n\n# running in google colab\nif 'google.colab' in str(get_ipython()):\n    NUM_EPOCHS = 50\n    !pip install torchviz\n    BASE_PATH = './drive/MyDrive/Colab/data/'\n    from google.colab import drive\n    drive.mount('/content/drive')\n    \n# running interactively in kaggle\nelif get_ipython().config.IPKernelApp.connection_file.startswith('/root/.local/share'):\n    NUM_EPOCHS = 5\n    BASE_PATH = '/kaggle/input/'\n    !pip install torchviz\n    \n# running as background job in kaggle\nelif 'SHLVL' in os.environ:\n    NUM_EPOCHS = 120\n    BASE_PATH = '/kaggle/input/'\n    !pip install torchviz\n\nelse:\n    BASE_PATH = '../data/'\n    NUM_EPOCHS = 2","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:12.288400Z","iopub.execute_input":"2023-09-20T09:30:12.289112Z","iopub.status.idle":"2023-09-20T09:30:35.355583Z","shell.execute_reply.started":"2023-09-20T09:30:12.289074Z","shell.execute_reply":"2023-09-20T09:30:35.354384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nfrom tqdm.auto import tqdm\nimport numpy as np\nfrom collections.abc import Callable\nimport locale\nlocale.setlocale(locale.LC_ALL, locale='')  # for thousands separator via ... print(f'{value:n}')\"\nimport math\nfrom itertools import islice\nfrom collections.abc import Iterable, Generator\nfrom pprint import pprint\nimport pathlib\nimport re\n\nfrom IPython.display import HTML, Image\nimport time\nimport matplotlib.animation as animation\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\nimport pandas as pd\nimport torch\nfrom torch import nn\nfrom torchvision import transforms\nimport torchvision\nfrom torchvision import datasets\nfrom torchvision.utils import make_grid\nfrom torchvision import utils\nfrom torch.utils.data import DataLoader\nfrom torch.nn.modules.loss import _Loss\nfrom torchviz import make_dot\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport sklearn.metrics\nimport cv2\n\nmy_seed = 123\nrandom.seed(my_seed)\ntorch.manual_seed(my_seed)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:35.359153Z","iopub.execute_input":"2023-09-20T09:30:35.360103Z","iopub.status.idle":"2023-09-20T09:30:37.080009Z","shell.execute_reply.started":"2023-09-20T09:30:35.360063Z","shell.execute_reply":"2023-09-20T09:30:37.079082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Zipped Images if Required","metadata":{}},{"cell_type":"code","source":"train_path = \"/kaggle/working/train/\"\nif not (\n    pathlib.Path(train_path).exists()\n    and len(list(pathlib.Path(train_path).iterdir())) == 50000\n):\n    \n    try:\n        import py7zr\n    except ModuleNotFoundError:\n        !pip install py7zr\n        import py7zr\n    \n    with py7zr.SevenZipFile('/kaggle/input/cifar-10/train.7z', mode='r') as z:\n        z.extractall(path=\"/kaggle/working/\")\n    print('finished downloading train')\n    \n    assert len(list(pathlib.Path(train_path).iterdir())) == 50000","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:37.081308Z","iopub.execute_input":"2023-09-20T09:30:37.082351Z","iopub.status.idle":"2023-09-20T09:30:37.317613Z","shell.execute_reply.started":"2023-09-20T09:30:37.082292Z","shell.execute_reply":"2023-09-20T09:30:37.316610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"/kaggle/working/test/\"\nif not (\n    pathlib.Path(test_path).exists()\n    and len(list(pathlib.Path(test_path).iterdir())) == 300000\n):\n\n    try:\n        import py7zr\n    except ModuleNotFoundError:\n        !pip install py7zr\n        import py7zr\n\n    # takes more than half an hour...\n    with py7zr.SevenZipFile('/kaggle/input/cifar-10/test.7z', mode='r') as z:\n        z.extractall(path=\"/kaggle/working/\")\n    print('finished downloading test')\n\n    assert len(list(pathlib.Path(test_path).iterdir())) == 300000","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:40.111063Z","iopub.execute_input":"2023-09-20T09:30:40.111475Z","iopub.status.idle":"2023-09-20T09:30:41.383616Z","shell.execute_reply.started":"2023-09-20T09:30:40.111441Z","shell.execute_reply":"2023-09-20T09:30:41.381697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = pathlib.Path('/kaggle/working/')\nfor p in path.iterdir():\n    print(p)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:42.318786Z","iopub.execute_input":"2023-09-20T09:30:42.319146Z","iopub.status.idle":"2023-09-20T09:30:42.325227Z","shell.execute_reply.started":"2023-09-20T09:30:42.319109Z","shell.execute_reply":"2023-09-20T09:30:42.324300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"# Read Labels","metadata":{"execution":{"iopub.status.busy":"2023-09-19T13:51:39.847012Z","iopub.execute_input":"2023-09-19T13:51:39.847363Z","iopub.status.idle":"2023-09-19T13:51:39.853399Z","shell.execute_reply.started":"2023-09-19T13:51:39.847336Z","shell.execute_reply":"2023-09-19T13:51:39.852485Z"}}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/cifar-10/trainLabels.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:43.267260Z","iopub.execute_input":"2023-09-20T09:30:43.268128Z","iopub.status.idle":"2023-09-20T09:30:43.309146Z","shell.execute_reply.started":"2023-09-20T09:30:43.268085Z","shell.execute_reply":"2023-09-20T09:30:43.308197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.describe())\ndf_train['label'].value_counts().plot(kind='bar')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:43.482781Z","iopub.execute_input":"2023-09-20T09:30:43.483267Z","iopub.status.idle":"2023-09-20T09:30:43.840169Z","shell.execute_reply.started":"2023-09-20T09:30:43.483238Z","shell.execute_reply":"2023-09-20T09:30:43.839225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Images from File System","metadata":{}},{"cell_type":"code","source":"# each id matches the image's filename: 1 -> \"1.png\" \ndf_train['filename'] = df_train['id'].astype(str) + '.png'\ndf_train","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:44.259524Z","iopub.execute_input":"2023-09-20T09:30:44.259889Z","iopub.status.idle":"2023-09-20T09:30:44.321928Z","shell.execute_reply.started":"2023-09-20T09:30:44.259859Z","shell.execute_reply":"2023-09-20T09:30:44.320909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the images are all 32x32 pixels with 3 color channels, no need to resize\nimage_list: list[np.array] = []\n\nfor i, filename in enumerate(df_train['filename']):\n    image = cv2.imread(train_path + filename,  # returns np.array of differentshape and dtype uint8\n                       flags=cv2.IMREAD_COLOR)  # read as 3 channel BGR (Blue-Green-Red)\n    image = cv2.cvtColor(src=image, \n                         code=cv2.COLOR_BGR2RGB)  # convert from BGR to RGB (Red-Green-Blue)\n    image_list.append(image)\n\n# merge into one array\nimages = np.asarray(image_list)  # (50000, 32, 32, 3), uint8\nassert len(images) == 50000\n\nprint(f'Memory Consumption of resized images: {images.nbytes :n}')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:44.645133Z","iopub.execute_input":"2023-09-20T09:30:44.645779Z","iopub.status.idle":"2023-09-20T09:30:49.126425Z","shell.execute_reply.started":"2023-09-20T09:30:44.645741Z","shell.execute_reply":"2023-09-20T09:30:49.125390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preview Some Images","metadata":{"execution":{"iopub.status.busy":"2023-09-19T14:03:54.867400Z","iopub.execute_input":"2023-09-19T14:03:54.867795Z","iopub.status.idle":"2023-09-19T14:03:54.875771Z","shell.execute_reply.started":"2023-09-19T14:03:54.867752Z","shell.execute_reply":"2023-09-19T14:03:54.874560Z"}}},{"cell_type":"code","source":"random_indexes = [random.randint(0, len(images)) for _ in range(20)]\nrandom_images = images[random_indexes]  # (20, 32, 32, 3)\nrandom_labels = df_train['label'][random_indexes]\n\nfig, axes = plt.subplots(nrows=5, \n                         ncols=4, \n                         figsize=(20, 20))\n\nfor i, (image, label) in enumerate(zip(random_images, random_labels)):\n\n    ax = axes[i//4, i%4]\n    ax.imshow(image)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_title(label)\n\nfig.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:49.128481Z","iopub.execute_input":"2023-09-20T09:30:49.129084Z","iopub.status.idle":"2023-09-20T09:30:50.619253Z","shell.execute_reply.started":"2023-09-20T09:30:49.129049Z","shell.execute_reply":"2023-09-20T09:30:50.618267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize\nNN work better with normalized [0.0...1.0] data as input instead of RGB [0...255].","metadata":{}},{"cell_type":"code","source":"normalized_images = images / 255  # (50000, 32, 32, 3), uint8 -> float64\nnormalized_images = normalized_images.astype(np.float32)  # float64 -> float32 (required later for torch)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:50.620368Z","iopub.execute_input":"2023-09-20T09:30:50.620920Z","iopub.status.idle":"2023-09-20T09:30:51.257426Z","shell.execute_reply.started":"2023-09-20T09:30:50.620887Z","shell.execute_reply":"2023-09-20T09:30:51.256470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encode Labels","metadata":{}},{"cell_type":"code","source":"label_encoder = sklearn.preprocessing.LabelEncoder()\n\ny = label_encoder.fit_transform(df_train['label'])  # np.array (50000,), int64, 0..9\n\nexample = ['dog']\nprint(f'Example: {example} -> {(enc := label_encoder.transform(example))} -> {label_encoder.inverse_transform(enc)}')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:51.260018Z","iopub.execute_input":"2023-09-20T09:30:51.260499Z","iopub.status.idle":"2023-09-20T09:30:51.284892Z","shell.execute_reply.started":"2023-09-20T09:30:51.260465Z","shell.execute_reply":"2023-09-20T09:30:51.283902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train / Validation Split","metadata":{}},{"cell_type":"code","source":"# we have images and labels in separate arrays\n# we need to randomize then synchronously\nrandomized_indexes = pd.Series(range(len(images))).sample(frac=1).index\ny_randomized = y[randomized_indexes]  # still np.array (50000,)\nimages_randomized = normalized_images[randomized_indexes]  # still np.array (50000, 32, 32, 3)\n\n# make sure we didn't get something wrong...\nrnd_iloc = 12345\nrnd_index = randomized_indexes[rnd_iloc]\nassert y_randomized[rnd_iloc] == y[rnd_index]\nassert (images_randomized[rnd_iloc] == normalized_images[rnd_index]).all()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:51.286367Z","iopub.execute_input":"2023-09-20T09:30:51.286874Z","iopub.status.idle":"2023-09-20T09:30:51.481161Z","shell.execute_reply.started":"2023-09-20T09:30:51.286843Z","shell.execute_reply":"2023-09-20T09:30:51.480283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = int(len(images_randomized) * 0.85)\n\nx_train_arr = images_randomized[:NUM_TRAIN]  # (42500, 32, 32, 3), float32\nx_val_arr = images_randomized[NUM_TRAIN:]  # (7500, 32, 32, 3)\n\ny_train_arr = y_randomized[:NUM_TRAIN]  # (42500,), int64\ny_val_arr = y_randomized[NUM_TRAIN:]  # (7500),)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:51.482943Z","iopub.execute_input":"2023-09-20T09:30:51.483339Z","iopub.status.idle":"2023-09-20T09:30:51.489680Z","shell.execute_reply.started":"2023-09-20T09:30:51.483292Z","shell.execute_reply":"2023-09-20T09:30:51.488744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tensorize and Create DataLoader","metadata":{}},{"cell_type":"code","source":"x_train = torch.tensor(x_train_arr).to(DEVICE)  # [42500, 32, 32, 3], torch.float32\nx_val = torch.tensor(x_val_arr).to(DEVICE)\n\ny_train = torch.tensor(y_train_arr).to(DEVICE)  # [42500], torch.int64\ny_val = torch.tensor(y_val_arr).to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:51.490856Z","iopub.execute_input":"2023-09-20T09:30:51.492079Z","iopub.status.idle":"2023-09-20T09:30:55.145064Z","shell.execute_reply.started":"2023-09-20T09:30:51.492017Z","shell.execute_reply":"2023-09-20T09:30:55.144063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 32\n\ntrain_dataset = torch.utils.data.TensorDataset(x_train, \n                                               y_train)\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, \n                                           batch_size=BATCH_SIZE, \n                                           shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:55.146539Z","iopub.execute_input":"2023-09-20T09:30:55.147153Z","iopub.status.idle":"2023-09-20T09:30:55.153693Z","shell.execute_reply.started":"2023-09-20T09:30:55.147118Z","shell.execute_reply":"2023-09-20T09:30:55.152711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class CNNClassifier(torch.nn.Module):\n\n    def __init__(self, \n                 dropout_probability=0.3,\n                 num_labels=12):\n        super(CNNClassifier, self).__init__()\n        \n        self.layer1 = torch.nn.Sequential(\n            # (batch_size, 3, 32, 32) -> [batch_size, 32, 32, 32]\n            torch.nn.Conv2d(in_channels=3,  # Number of channels in the input image\n                            out_channels=32,  # Number of channels produced by the convolution\n                            kernel_size=3, #  Size of the convolving kernel\n                            stride=1,  # Stride of the convolution. Default: 1\n                            padding=1,  # Padding added to all four sides of the input. Default: 0\n                           ),\n            # (element-wise)\n            torch.nn.ReLU(),\n            # same shape\n            torch.nn.BatchNorm2d(num_features=32),\n            # [batch_size, 32, 32, 32]  - > [batch_size, 32, 16, 16]\n            torch.nn.MaxPool2d(kernel_size=2, # the size of the window to take a max over\n                               stride=2,  # the stride of the window. Default value is kernel_size\n                              ),\n            # (element-wise)\n            torch.nn.Dropout(p=dropout_probability,  # probability of an element to be zeroed. Default: 0.5\n                            ),\n        )\n\n        self.layer2 = torch.nn.Sequential(\n            # [batch_size, 32, 16, 16] --> [batch_size, 64, 16, 16]\n            torch.nn.Conv2d(in_channels=32,\n                            out_channels=64,\n                            kernel_size=3,\n                            stride=1,\n                            padding=1),\n            torch.nn.ReLU(),\n            # same shape\n            torch.nn.BatchNorm2d(num_features=64),  # significantly improves performance\n            # [batch_size, 64, 16, 16] --> [batch_size, 64, 8, 8]\n            torch.nn.MaxPool2d(kernel_size=2,\n                               stride=2),\n            torch.nn.Dropout(p=dropout_probability))\n\n        self.layer3 = torch.nn.Sequential(\n            # [batch_size, 64, 8, 8] --> [batch_size, 128, 8, 8]\n            torch.nn.Conv2d(in_channels=64,\n                            out_channels=128,\n                            kernel_size=3,\n                            stride=1,\n                            padding=1),\n            torch.nn.ReLU(),\n            # same shape\n            torch.nn.BatchNorm2d(num_features=128),\n            # [batch_size, 128, 8, 8] --> [batch_size, 128, 5, 5]\n            torch.nn.MaxPool2d(kernel_size=2, \n                               stride=2, \n                               padding=1),  # default: 0\n            torch.nn.Dropout(p=dropout_probability)\n            )\n        \n        # [batch_size, 128, 5, 5] -> [batch_size, 3200]\n        self.flatten = torch.nn.Flatten()  # for feed-forward \n\n# at1 and mat2 shapes cannot be multiplied (16x10368 and 2048x625)\n        \n        # [batch_size, 3200] --> [batch_size, 625]\n        self.fc1 = torch.nn.Linear(in_features=5 * 5 * 128,\n                                   out_features=625,\n                                   bias=True)\n        \n        # [batch_size, 625] --> [batch_size, 12]\n        self.fc2 = torch.nn.Linear(in_features=625,\n                                   out_features=num_labels,\n                                   bias=True)\n        \n        torch.nn.init.xavier_uniform_(self.fc1.weight)  # initialize weights (seems to make no difference)\n        torch.nn.init.xavier_uniform_(self.fc2.weight) \n        \n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:  # x: [batch_size, 32, 32, 3], torch.float32\n        \n        # we need the channels at the beginning\n        # [batch_size, 32, 32, 3] -> [batch_size, 3, 32, 32]\n        x = x.permute(dims=(0, 3, 1, 2))\n        \n        # CNN\n        output_layer_1 = self.layer1(x)   # [batch_size, 32, 16, 16]\n        output_layer_2 = self.layer2(output_layer_1)  # [batch_size, 64, 8, 8]\n        output_layer_3 = self.layer3(output_layer_2)  # [batch_size, 128, 5, 5]\n        flattened = self.flatten(output_layer_3)  # [batch_size, 3200]\n        \n        # FC\n        output_fully_connected_1 = self.fc1(flattened)  # [batch_size, 625]\n        output_fully_connected_2 = self.fc2(output_fully_connected_1)  # [batch_size, 12]\n\n        return output_fully_connected_2","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:55.155231Z","iopub.execute_input":"2023-09-20T09:30:55.155822Z","iopub.status.idle":"2023-09-20T09:30:55.172649Z","shell.execute_reply.started":"2023-09-20T09:30:55.155790Z","shell.execute_reply":"2023-09-20T09:30:55.171669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize the classifier and make sure it generally works\nc_temp = CNNClassifier().to(DEVICE)\n# to visualize with torchviz, we need some input that can pass through the model's forward() method.\nx_batch, _ = next(iter(train_loader)) \npredictions = c_temp(x_batch)\nmake_dot(predictions)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:30:55.176480Z","iopub.execute_input":"2023-09-20T09:30:55.176741Z","iopub.status.idle":"2023-09-20T09:31:01.557371Z","shell.execute_reply.started":"2023-09-20T09:30:55.176718Z","shell.execute_reply":"2023-09-20T09:31:01.556364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train","metadata":{}},{"cell_type":"code","source":"LEARNING_RATE = 0.001\n\nclassifier = CNNClassifier().to(DEVICE)\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(classifier.parameters(),\n                             lr = LEARNING_RATE)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,  # reduce learning rate when model stops improving on validation dataset \n                                                       mode='min', \n                                                       verbose=True)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:01.558745Z","iopub.execute_input":"2023-09-20T09:31:01.561618Z","iopub.status.idle":"2023-09-20T09:31:01.643059Z","shell.execute_reply.started":"2023-09-20T09:31:01.561580Z","shell.execute_reply":"2023-09-20T09:31:01.641829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metrics(classifier: CNNClassifier, \n                    loss_fn: Callable,\n                    x: torch.Tensor, \n                    y: torch.Tensor\n                   )->tuple[float, float, float]:\n    \n        y_pred_logits = classifier(x)\n        loss = loss_fn(y_pred_logits, y).item()\n    \n        y_pred = y_pred_logits.argmax(dim=1)\n        correct = (y_pred == y).type(torch.FloatTensor)\n        accuracy = correct.mean().item()\n\n        f1_score = sklearn.metrics.f1_score(y_true=y.cpu(), \n                                            y_pred=y_pred.cpu(),\n                                            average='micro')  # multi-class problem\n        \n        return loss, accuracy, f1_score","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:01.646229Z","iopub.execute_input":"2023-09-20T09:31:01.647591Z","iopub.status.idle":"2023-09-20T09:31:01.659946Z","shell.execute_reply.started":"2023-09-20T09:31:01.647556Z","shell.execute_reply":"2023-09-20T09:31:01.657575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_metrics = pd.DataFrame(columns=['loss_train', 'accuracy_train', 'f1_train', \n                                   'loss_val', 'accuracy_val', 'f1_val'],\n                          index=range(NUM_EPOCHS),\n                          dtype=float)\n\nfor epoch in tqdm(range(NUM_EPOCHS)):\n\n    for batch, (x_train_batch, y_train_batch) in enumerate(train_loader):\n        # x_train_batch: [batch_size, 70, 70, 3] torch.float32\n        # y_train_batch: [batch_size] torch.int64\n\n        # x_train_batch = x_train_batch.to(DEVICE)\n        # y_train_batch = y_train_batch.to(DEVICE)\n\n        # switch to training mode mode (we might have been in evaluation mode)\n        classifier.train()\n\n        pred_train_batch_logits = classifier(x_train_batch)  # [batch_size, 12], float32\n\n        # clear existing gradients from previous batch\n        optimizer.zero_grad()\n        loss = loss_fn(pred_train_batch_logits,\n                       y_train_batch)  # [], .item() is e.g. 2.291177988052368\n\n        # compute gradients (backpropagation), then apply gradients\n        loss.backward()\n        optimizer.step()\n        \n    # after each epoch, switch to evaluation mode, then evaluate without computing gradients\n    classifier.eval()\n    with torch.no_grad():\n        loss_train, accuracy_train, f1_score_train = compute_metrics(classifier, loss_fn, x_train, y_train)\n        loss_val, accuracy_val, f1_score_val = compute_metrics(classifier, loss_fn, x_val, y_val)\n\n        df_metrics.iloc[epoch] = [loss_train, accuracy_train, f1_score_train,\n                                  loss_val, accuracy_val, f1_score_val]\n        \n    scheduler.step(loss_val)\n    print(f'Accuracy Validation after epoch {epoch}: {accuracy_val :.4f}  '\n          f'(Train: {accuracy_train :.4f}) '\n          f'LR = {optimizer.param_groups[0][\"lr\"]}\\n')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:01.664766Z","iopub.execute_input":"2023-09-20T09:31:01.667122Z","iopub.status.idle":"2023-09-20T09:31:32.371989Z","shell.execute_reply.started":"2023-09-20T09:31:01.667090Z","shell.execute_reply":"2023-09-20T09:31:32.371060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation","metadata":{}},{"cell_type":"code","source":"df_metrics.style.background_gradient(cmap='Blues')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:32.373541Z","iopub.execute_input":"2023-09-20T09:31:32.374484Z","iopub.status.idle":"2023-09-20T09:31:32.473401Z","shell.execute_reply.started":"2023-09-20T09:31:32.374449Z","shell.execute_reply":"2023-09-20T09:31:32.472276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = range(NUM_EPOCHS)\n\nfig, ((ax1, ax2), (ax3, _)) = plt.subplots(nrows=2,\n                                       ncols=2,\n                                       figsize=(15,5),\n                                          sharex=True)\n\n# Plot and label the training and val loss values\nax1.plot(epochs, df_metrics['loss_train'], label='Training Loss')\nax1.plot(epochs, df_metrics['loss_val'], label='val Loss')\nax1.set_ylabel('Loss')\nax1.legend(loc='best')\n\n# ... Accuracy\nax2.plot(epochs, df_metrics['accuracy_train'], label='Training Accuracy')\nax2.plot(epochs, df_metrics['accuracy_val'], label='val Accuracy')\nax2.set_ylabel('Accuracy')\nax2.legend(loc='best')\n\n# ... F1-Score\nax3.plot(epochs, df_metrics['f1_train'], label='Training F1-Score')\nax3.plot(epochs, df_metrics['f1_val'], label='val F1-Score')\nax3.set_ylabel('F1-Score')\nax3.legend(loc='best')\nax3.set_xlabel('Epochs')\nax3.set_xticks(np.arange(0, \n                         NUM_EPOCHS))\n\nplt.suptitle('Training and Validation Metrics')\nplt.xlabel('Epochs')\nplt.xticks(np.arange(0, \n                     NUM_EPOCHS))\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:32.474658Z","iopub.execute_input":"2023-09-20T09:31:32.475102Z","iopub.status.idle":"2023-09-20T09:31:33.402957Z","shell.execute_reply.started":"2023-09-20T09:31:32.475070Z","shell.execute_reply":"2023-09-20T09:31:33.400403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\nPredict all 300.000 test images (10.000 real images, 290.000 junk images)","metadata":{}},{"cell_type":"code","source":"paths = [str(path) for path in pathlib.Path(test_path).iterdir()]\n\nregex = '[^\\/]+(?=\\.png$)'\npaths.sort(key= lambda path: int(re.search(regex, path, re.IGNORECASE).group(0)))\n\n# we need to load images and predict in chunks\ndef chunkify(l: list, n: int):\n    for i in range(0, len(l), n): \n        yield l[i:i + n]\n\nchunks = chunkify(paths, 100)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:33.407188Z","iopub.execute_input":"2023-09-20T09:31:33.409475Z","iopub.status.idle":"2023-09-20T09:31:36.750164Z","shell.execute_reply.started":"2023-09-20T09:31:33.409440Z","shell.execute_reply":"2023-09-20T09:31:36.749191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = []\n\nfor chunk in chunks:\n    chunk: list[str]\n        \n    image_list = []\n    for path in chunk:\n        image = cv2.imread(path,\n                           flags=cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(src=image, \n                             code=cv2.COLOR_BGR2RGB)\n        image_list.append(image)\n        \n    images = np.asarray(image_list)  # (100, 32, 32, 3), uint8\n    \n    normalized_images = images / 255  # (100, 32, 32, 3), uint8 -> float64\n    normalized_images = normalized_images.astype(np.float32)  # float64 -> float32\n        \n    x_test = torch.tensor(normalized_images).to(DEVICE)  # [100, 32, 32, 3], torch.float32\n        \n    pred_logits = classifier(x_test)\n    pred = pred_logits.argmax(dim=1)\n    \n    pred_labels = label_encoder.inverse_transform(pred.cpu()).tolist()\n    predictions.extend(pred_labels)","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:31:39.454446Z","iopub.execute_input":"2023-09-20T09:31:39.454802Z","iopub.status.idle":"2023-09-20T09:32:18.902386Z","shell.execute_reply.started":"2023-09-20T09:31:39.454775Z","shell.execute_reply":"2023-09-20T09:32:18.901375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this is how the submission file must look like\nwith open(BASE_PATH + \"cifar-10/sampleSubmission.csv\")as f:\n    print(f.readline())\n    print(f.readline())\n    print(f.readline())\n    print('...')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T09:40:50.264072Z","iopub.execute_input":"2023-09-20T09:40:50.264480Z","iopub.status.idle":"2023-09-20T09:40:50.271169Z","shell.execute_reply.started":"2023-09-20T09:40:50.264448Z","shell.execute_reply":"2023-09-20T09:40:50.270045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unfortunately, it seems like if the working dir has a certain amount of data (or files),\n# the output is zipped, including submission.csv. submission.csv is therefor not found anymore.\n# when submitting, we need to remove (at least) the test images (300.000).\n# todo: can we find out we're in a submission run???\n\nimport shutil\nshutil.rmtree('/kaggle/working/test')\nassert not pathlib.Path('/kaggle/working/test').exists()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assert len(paths) == len(predictions)\n\n# extract ids from paths/filenames\nids = [re.search(regex, path, re.IGNORECASE).group(0) for path in paths]\n\ndf_submission = pd.DataFrame({'id': ids,\n                              'label': predictions})\ndf_submission.to_csv(\"submission.csv\", \n                     index=False)\n\n# make sure our file looks as expected\nwith open(\"submission.csv\")as f:\n    print(f.readline())\n    print(f.readline())\n    print(f.readline())\n    print('...')","metadata":{"execution":{"iopub.status.busy":"2023-09-20T08:31:42.248320Z","iopub.execute_input":"2023-09-20T08:31:42.248708Z","iopub.status.idle":"2023-09-20T08:31:45.084223Z","shell.execute_reply.started":"2023-09-20T08:31:42.248678Z","shell.execute_reply":"2023-09-20T08:31:45.083146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}