{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":65471,"databundleVersionId":7211464,"sourceType":"competition"}],"dockerImageVersionId":30588,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%capture\n\n!pip install einops","metadata":{"execution":{"iopub.status.busy":"2023-12-23T17:57:12.659814Z","iopub.execute_input":"2023-12-23T17:57:12.660845Z","iopub.status.idle":"2023-12-23T17:57:26.055525Z","shell.execute_reply.started":"2023-12-23T17:57:12.660810Z","shell.execute_reply":"2023-12-23T17:57:26.054400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport torch\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\n\nfrom torchvision import transforms\nfrom torchvision.models import resnet50\nfrom torch.utils.data import Dataset, DataLoader, random_split,ConcatDataset\n\nfrom sklearn.metrics import confusion_matrix\n\nfrom PIL import Image\nimport seaborn as sns\n\nimport csv\nfrom tqdm import tqdm\n\n%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:31:54.556426Z","iopub.execute_input":"2023-12-23T21:31:54.557416Z","iopub.status.idle":"2023-12-23T21:31:54.570448Z","shell.execute_reply.started":"2023-12-23T21:31:54.557378Z","shell.execute_reply":"2023-12-23T21:31:54.569478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def nameToLabel(name):\n  return \"_\".join(name.replace('.', '_').split('_')[1:3])\n\ndef imshow(image, ax=None, title=None, normalize=True):\n    if ax is None:\n        fig, ax = plt.subplots()\n    image = image.numpy().transpose((1, 2, 0))\n\n    if normalize:\n        mean = np.array([0.485, 0.456, 0.406])\n        std = np.array([0.229, 0.224, 0.225])\n        image = std * image + mean\n        image = np.clip(image, 0, 1)\n\n    ax.imshow(image, cmap='Greys_r')\n    ax.spines['top'].set_visible(False)\n    ax.spines['right'].set_visible(False)\n    ax.spines['left'].set_visible(False)\n    ax.spines['bottom'].set_visible(False)\n    ax.set_title(title)\n    ax.tick_params(axis='both', length=0)\n    ax.set_xticklabels('')\n    ax.set_yticklabels('')\n\n    return ax","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:31:59.955367Z","iopub.execute_input":"2023-12-23T21:31:59.956095Z","iopub.status.idle":"2023-12-23T21:31:59.964300Z","shell.execute_reply.started":"2023-12-23T21:31:59.956061Z","shell.execute_reply":"2023-12-23T21:31:59.963350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Implementation of Resnet50 from Scratch in a simple way\n\nclass Bottleneck(nn.Module):\n    expansion = 4\n\n    def __init__(self, in_channels, out_channels, stride=1):\n        super(Bottleneck, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        self.conv3 = nn.Conv2d(out_channels, out_channels * self.expansion, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(out_channels * self.expansion)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = nn.Sequential()\n        if stride != 1 or in_channels != out_channels * self.expansion:\n            self.downsample = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels * self.expansion, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels * self.expansion)\n            )\n\n    def forward(self, x):\n        identity = x\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n        out = self.conv3(out)\n        out = self.bn3(out)\n        out += self.downsample(identity)\n        out = self.relu(out)\n        return out\n\nclass ResNet50(nn.Module):\n    def __init__(self, block, layers, num_classes=65, dropout_prob=0.5):\n        super(ResNet50, self).__init__()\n        self.in_channels = 64\n        self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3, bias=False)\n        self.bn1 = nn.BatchNorm2d(64)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n        self.dropout = nn.Dropout(p=dropout_prob)\n\n    def _make_layer(self, block, out_channels, blocks, stride=1):\n        layers = []\n        layers.append(block(self.in_channels, out_channels, stride))\n        self.in_channels = out_channels * block.expansion\n        for _ in range(1, blocks):\n            layers.append(block(out_channels * block.expansion, out_channels, stride=1))\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        \n        x = self.avgpool(x)\n        x = self.dropout(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:00.484497Z","iopub.execute_input":"2023-12-23T21:32:00.485051Z","iopub.status.idle":"2023-12-23T21:32:00.505838Z","shell.execute_reply.started":"2023-12-23T21:32:00.485020Z","shell.execute_reply":"2023-12-23T21:32:00.504922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Implementation of Vision Transformer with Patch Dropout\n\n# from einops import rearrange\n# from einops.layers.torch import Rearrange\n\n# def pair(t):\n#     return t if isinstance(t, tuple) else (t, t)\n\n# def posemb_sincos_2d(patches, temperature = 10000, dtype = torch.float32):\n#     _, h, w, dim, device, dtype = *patches.shape, patches.device, patches.dtype\n\n#     y, x = torch.meshgrid(torch.arange(h, device = device), torch.arange(w, device = device), indexing = 'ij')\n#     assert (dim % 4) == 0, 'feature dimension must be multiple of 4 for sincos emb'\n#     omega = torch.arange(dim // 4, device = device) / (dim // 4 - 1)\n#     omega = 1. / (temperature ** omega)\n\n#     y = y.flatten()[:, None] * omega[None, :]\n#     x = x.flatten()[:, None] * omega[None, :] \n#     pe = torch.cat((x.sin(), x.cos(), y.sin(), y.cos()), dim = 1)\n#     return pe.type(dtype)\n\n# class PatchDropout(nn.Module):\n#     def __init__(self, prob):\n#         super().__init__()\n#         assert 0 <= prob < 1.\n#         self.prob = prob\n\n#     def forward(self, x):\n#         if not self.training or self.prob == 0.:\n#             return x\n\n#         b, n, _, device = *x.shape, x.device\n\n#         batch_indices = torch.arange(b, device = device)\n#         batch_indices = rearrange(batch_indices, '... -> ... 1')\n#         num_patches_keep = max(1, int(n * (1 - self.prob)))\n#         patch_indices_keep = torch.randn(b, n, device = device).topk(num_patches_keep, dim = -1).indices\n\n#         return x[batch_indices, patch_indices_keep]\n\n# # classes\n\n# class FeedForward(nn.Module):\n#     def __init__(self, dim, hidden_dim):\n#         super().__init__()\n#         self.net = nn.Sequential(\n#             nn.LayerNorm(dim),\n#             nn.Linear(dim, hidden_dim),\n#             nn.GELU(),\n#             nn.Linear(hidden_dim, dim),\n#         )\n#     def forward(self, x):\n#         return self.net(x)\n\n# class Attention(nn.Module):\n#     def __init__(self, dim, heads = 8, dim_head = 64):\n#         super().__init__()\n#         inner_dim = dim_head *  heads\n#         self.heads = heads\n#         self.scale = dim_head ** -0.5\n#         self.norm = nn.LayerNorm(dim)\n\n#         self.attend = nn.Softmax(dim = -1)\n\n#         self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n#         self.to_out = nn.Linear(inner_dim, dim, bias = False)\n\n#     def forward(self, x):\n#         x = self.norm(x)\n\n#         qkv = self.to_qkv(x).chunk(3, dim = -1)\n#         q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)\n\n#         dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n\n#         attn = self.attend(dots)\n\n#         out = torch.matmul(attn, v)\n#         out = rearrange(out, 'b h n d -> b n (h d)')\n#         return self.to_out(out)\n# class Transformer(nn.Module):\n#     def __init__(self, dim, depth, heads, dim_head, mlp_dim):\n#         super().__init__()\n#         self.norm = nn.LayerNorm(dim)\n#         self.layers = nn.ModuleList([])\n#         for _ in range(depth):\n#             self.layers.append(nn.ModuleList([\n#                 Attention(dim, heads = heads, dim_head = dim_head),\n#                 FeedForward(dim, mlp_dim)\n#             ]))\n#     def forward(self, x):\n#         for attn, ff in self.layers:\n#             x = attn(x) + x\n#             x = ff(x) + x\n#         return self.norm(x)\n\n# class SimpleViT(nn.Module):\n#     def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, channels = 3, dim_head = 64, patch_dropout = 0.5):\n#         super().__init__()\n#         image_height, image_width = pair(image_size)\n#         patch_height, patch_width = pair(patch_size)\n\n#         assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'\n\n#         num_patches = (image_height // patch_height) * (image_width // patch_width)\n#         patch_dim = channels * patch_height * patch_width\n\n#         self.to_patch_embedding = nn.Sequential(\n#             Rearrange('b c (h p1) (w p2) -> b h w (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n#             nn.LayerNorm(patch_dim),\n#             nn.Linear(patch_dim, dim),\n#             nn.LayerNorm(dim)\n#         )\n\n#         self.patch_dropout = PatchDropout(patch_dropout)\n\n#         self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim)\n\n#         self.to_latent = nn.Identity()\n#         self.linear_head = nn.Linear(dim, num_classes)\n\n#     def forward(self, img):\n#         *_, h, w, dtype = *img.shape, img.dtype\n\n#         x = self.to_patch_embedding(img)\n#         pe = posemb_sincos_2d(x)\n#         x = rearrange(x, 'b ... d -> b (...) d') + pe\n\n#         x = self.patch_dropout(x)\n\n#         x = self.transformer(x)\n#         x = x.mean(dim = 1)\n\n#         x = self.to_latent(x)\n#         return self.linear_head(x)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T20:23:57.605360Z","iopub.execute_input":"2023-12-23T20:23:57.606047Z","iopub.status.idle":"2023-12-23T20:23:57.614285Z","shell.execute_reply.started":"2023-12-23T20:23:57.606015Z","shell.execute_reply":"2023-12-23T20:23:57.613394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CNN_Model(nn.Module):\n#     def __init__(self, num_classes=65, dropout_prob=0.5):\n#         super(CNN_Model, self).__init__()\n\n#         self.conv1 = nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=1)  # Adjust input channel to 1\n#         self.relu1 = nn.ReLU()\n#         self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n#         self.conv2 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1)\n#         self.relu2 = nn.ReLU()\n#         self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n#         self.conv3 = nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1)\n#         self.relu3 = nn.ReLU()\n#         self.maxpool3 = nn.MaxPool2d(kernel_size=2, stride=2)\n\n#         self.flatten = nn.Flatten()\n\n#         self.fc1 = nn.Linear(128 * 288, 512)\n#         self.relu4 = nn.ReLU()\n#         self.dropout = nn.Dropout(p=dropout_prob)\n\n#         self.fc2 = nn.Linear(512, num_classes)\n\n#     def forward(self, x):\n#         x = self.conv1(x)\n#         x = self.relu1(x)\n#         x = self.maxpool1(x)\n\n#         x = self.conv2(x)\n#         x = self.relu2(x)\n#         x = self.maxpool2(x)\n\n#         x = self.conv3(x)\n#         x = self.relu3(x)\n#         x = self.maxpool3(x)\n\n#         x = self.flatten(x)\n\n#         x = self.fc1(x)\n#         x = self.relu4(x)\n#         x = self.dropout(x)\n\n#         x = self.fc2(x)\n\n#         return x","metadata":{"execution":{"iopub.status.busy":"2023-12-23T20:26:32.006093Z","iopub.execute_input":"2023-12-23T20:26:32.006456Z","iopub.status.idle":"2023-12-23T20:26:32.017989Z","shell.execute_reply.started":"2023-12-23T20:26:32.006426Z","shell.execute_reply":"2023-12-23T20:26:32.016870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_folder = \"/kaggle/input/arabic-letters-classification/Final_Arabic_Alpha_dataset/Final_Arabic_Alpha_dataset/train/\"\nlabels_dict = {}\n\nfor category in sorted(os.listdir(base_folder), key=lambda x: int(x)):\n    for picture in os.listdir(base_folder + category):\n      labels_dict[category] = nameToLabel(picture)\n      break\nlabels_dict","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:05.583258Z","iopub.execute_input":"2023-12-23T21:32:05.583638Z","iopub.status.idle":"2023-12-23T21:32:05.682325Z","shell.execute_reply.started":"2023-12-23T21:32:05.583605Z","shell.execute_reply":"2023-12-23T21:32:05.681455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, root_folder, transform=None):\n        self.root_folder = root_folder\n        self.transform = transform\n        self.classes = sorted(os.listdir(root_folder), key=lambda x: int(x))\n        self.class_to_idx = {cls: i for i, cls in enumerate(self.classes)}\n        self.images = self.make_dataset()\n\n    def make_dataset(self):\n        images = []\n        for label in self.classes:\n            label_folder = os.path.join(self.root_folder, label)\n            for image_name in os.listdir(label_folder):\n                image_path = os.path.join(label_folder, image_name)\n                item = (image_path, self.class_to_idx[label])\n                images.append(item)\n        return images\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, index):\n        image_path, label = self.images[index]\n        image = Image.open(image_path).convert(\"RGB\")\n\n        if self.transform:\n            image = self.transform(image)\n\n        return image, label","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:06.148838Z","iopub.execute_input":"2023-12-23T21:32:06.149167Z","iopub.status.idle":"2023-12-23T21:32:06.158611Z","shell.execute_reply.started":"2023-12-23T21:32:06.149142Z","shell.execute_reply":"2023-12-23T21:32:06.157691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Threshold(object):\n    def __init__(self, threshold):\n        self.threshold = threshold\n\n    def __call__(self, img):\n        return (img > self.threshold).float()","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:23.552466Z","iopub.execute_input":"2023-12-23T21:32:23.553112Z","iopub.status.idle":"2023-12-23T21:32:23.558146Z","shell.execute_reply.started":"2023-12-23T21:32:23.553077Z","shell.execute_reply":"2023-12-23T21:32:23.557213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.Resize((64, 64)),\n    transforms.Grayscale(num_output_channels=1),\n    transforms.ToTensor(),\n    Threshold(0.8)\n])\n\ndataset = CustomDataset(base_folder, transform=transform)\n\n# dataset = CustomDataset(base_folder, transform=transform)\n\ntrain_size = int(0.8 * len(dataset))\n\n# val_size = len(dataset) - train_size\nval_size = int(0.1 * len(dataset))\n\ntest_size = len(dataset) - train_size - val_size\n\ntrain_dataset, val_dataset, test_dataset = random_split(dataset, [train_size, val_size, test_size])\n# train_dataset, val_dataset = random_split(dataset, [train_size, val_size])","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:39:43.415633Z","iopub.execute_input":"2023-12-23T21:39:43.416484Z","iopub.status.idle":"2023-12-23T21:39:43.606213Z","shell.execute_reply.started":"2023-12-23T21:39:43.416448Z","shell.execute_reply":"2023-12-23T21:39:43.605453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 512\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\nval_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True)\ntest_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:39:44.304700Z","iopub.execute_input":"2023-12-23T21:39:44.305020Z","iopub.status.idle":"2023-12-23T21:39:44.313540Z","shell.execute_reply.started":"2023-12-23T21:39:44.304987Z","shell.execute_reply":"2023-12-23T21:39:44.312592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch_idx, (images, labels) in enumerate(train_dataloader):\n    if batch_idx == 0:\n        for i in range(min(2, batch_size)):\n            imshow(images[i], title=labels_dict[str(labels[i].item())], normalize=False)\n            print(f\"Label: {labels[i]}\")\n            print(images[i].shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:28.518064Z","iopub.execute_input":"2023-12-23T21:32:28.518849Z","iopub.status.idle":"2023-12-23T21:32:29.641221Z","shell.execute_reply.started":"2023-12-23T21:32:28.518820Z","shell.execute_reply":"2023-12-23T21:32:29.640371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch_idx, (images, labels) in enumerate(test_dataloader):\n    if batch_idx == 0:\n        for i in range(min(2, batch_size)):\n            imshow(images[i], title=labels_dict[str(labels[i].item())], normalize=False)\n            print(f\"Label: {labels[i]}\")\n            print(images[i].shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:32.434798Z","iopub.execute_input":"2023-12-23T21:32:32.435171Z","iopub.status.idle":"2023-12-23T21:32:33.683607Z","shell.execute_reply.started":"2023-12-23T21:32:32.435139Z","shell.execute_reply":"2023-12-23T21:32:33.682567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for batch_idx, (images, labels) in enumerate(val_dataloader):\n    if batch_idx == 0:\n        for i in range(min(2, batch_size)):\n            imshow(images[i], title=labels_dict[str(labels[i].item())], normalize=False)\n            print(f\"Label: {labels[i]}\")\n            print(images[i].shape)\n        break","metadata":{"execution":{"iopub.status.busy":"2023-12-23T21:32:33.685164Z","iopub.execute_input":"2023-12-23T21:32:33.685505Z","iopub.status.idle":"2023-12-23T21:32:34.744940Z","shell.execute_reply.started":"2023-12-23T21:32:33.685467Z","shell.execute_reply":"2023-12-23T21:32:34.743631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate = 0.001\ndropout_prob = 0.5\nnum_classes = len(dataset.classes)\n\nmodel = ResNet50(Bottleneck, [4, 6, 7, 4], num_classes=num_classes, dropout_prob=dropout_prob)\n\n# model = CNN_Model(num_classes=num_classes)\n\nstep_size = 10\ngamma = 0.1 \ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\nnum_epochs = 50\n\nscheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)\n\nif(torch.cuda.is_available()):\n    model = model.cuda()\n    \nif torch.cuda.device_count() > 1:\n    print(\"Using\", torch.cuda.device_count(), \"GPUs!\")\n    model = nn.DataParallel(model)","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:54:51.135469Z","iopub.execute_input":"2023-12-23T22:54:51.135877Z","iopub.status.idle":"2023-12-23T22:54:51.395071Z","shell.execute_reply.started":"2023-12-23T22:54:51.135844Z","shell.execute_reply":"2023-12-23T22:54:51.394133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loss_history = []\ntrain_accuracy_history = []\nval_loss_history = []\nval_accuracy_history = []","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:54:57.895410Z","iopub.execute_input":"2023-12-23T22:54:57.895833Z","iopub.status.idle":"2023-12-23T22:54:57.900982Z","shell.execute_reply.started":"2023-12-23T22:54:57.895798Z","shell.execute_reply":"2023-12-23T22:54:57.900037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    total_correct_train = 0\n    total_train = 0\n\n    with tqdm(total=len(train_dataloader), desc=f'Training - Epoch {epoch + 1}/{num_epochs}', unit='batch') as pbar:\n        for batch_idx, (inputs, labels) in enumerate(train_dataloader):\n            if torch.cuda.is_available():\n                inputs = inputs.cuda()\n                labels = labels.cuda()\n\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            torch.cuda.empty_cache()\n            running_loss += loss.item()\n\n            _, predicted_train = torch.max(outputs.data, 1)\n            total_correct_train += (predicted_train == labels).sum().item()\n            total_train += labels.size(0)\n\n            pbar.set_postfix(loss=loss.item(), acc=total_correct_train / total_train)\n            pbar.update()\n\n    model.eval()\n    val_loss = 0.0\n    correct = 0\n    total = 0\n\n    with tqdm(total=len(val_dataloader), desc=f'Validation - Epoch {epoch + 1}/{num_epochs}', unit='batch') as pbar:\n        with torch.no_grad():\n            for batch_idx, (val_inputs, val_labels) in enumerate(val_dataloader):\n                if torch.cuda.is_available():\n                    val_inputs = val_inputs.cuda()\n                    val_labels = val_labels.cuda()\n\n                val_outputs = model(val_inputs)\n                val_loss += criterion(val_outputs, val_labels).item()\n\n                _, predicted = torch.max(val_outputs.data, 1)\n                total += val_labels.size(0)\n                correct += (predicted == val_labels).sum().item()\n                torch.cuda.empty_cache()\n\n                pbar.set_postfix(loss=val_loss / (batch_idx + 1), acc=correct / total)\n                pbar.update()\n\n    avg_train_loss = running_loss / len(train_dataloader)\n    avg_val_loss = val_loss / len(val_dataloader)\n    train_accuracy = total_correct_train / total_train\n    val_accuracy = correct / total\n    \n    train_loss_history.append(avg_train_loss)\n    train_accuracy_history.append(train_accuracy)\n    val_loss_history.append(avg_val_loss)\n    val_accuracy_history.append(val_accuracy)\n    \n    scheduler.step()\n    \n    print(f\"Epoch {epoch + 1}/{num_epochs}, Average Training Loss: {round(avg_train_loss, 2)}, Average Validation Loss: {round(avg_val_loss, 2)}, Training Accuracy: {round(train_accuracy, 2)}, Validation Accuracy: {round(val_accuracy, 2)}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-23T22:54:59.289771Z","iopub.execute_input":"2023-12-23T22:54:59.290417Z","iopub.status.idle":"2023-12-24T00:25:14.665771Z","shell.execute_reply.started":"2023-12-23T22:54:59.290383Z","shell.execute_reply":"2023-12-24T00:25:14.664521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retraining the model on the validation data\n\nval_train_epochs = 10\n\nfor epoch in range(val_train_epochs):\n    model.train()\n    running_loss = 0.0\n    total_correct_train = 0\n    total_train = 0\n\n    with tqdm(total=len(val_dataloader), desc=f'Training on validation data - Epoch {epoch + 1}/{val_train_epochs}', unit='batch') as pbar:\n        for batch_idx, (inputs, labels) in enumerate(val_dataloader):\n            if torch.cuda.is_available():\n                inputs = inputs.cuda()\n                labels = labels.cuda()\n\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            torch.cuda.empty_cache()\n            running_loss += loss.item()\n\n            _, predicted_train = torch.max(outputs.data, 1)\n            total_correct_train += (predicted_train == labels).sum().item()\n            total_train += labels.size(0)\n\n            pbar.set_postfix(loss=loss.item(), acc=total_correct_train / total_train)\n            pbar.update()\n            \n    avg_train_loss = running_loss / len(train_dataloader)\n    train_accuracy = total_correct_train / total_train","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:36:27.593397Z","iopub.execute_input":"2023-12-24T00:36:27.593849Z","iopub.status.idle":"2023-12-24T00:38:32.508748Z","shell.execute_reply.started":"2023-12-24T00:36:27.593814Z","shell.execute_reply":"2023-12-24T00:38:32.507429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluating with test set\n\nmodel.eval()\ncorrect = 0\ntotal = 0\nall_labels = []\nall_preds = []\n\nwith torch.no_grad(), tqdm(total=len(test_dataloader), desc='Testing', unit='batch') as pbar:\n    for test_inputs, test_labels in test_dataloader:\n        if torch.cuda.is_available():\n            test_inputs = test_inputs.cuda()\n            test_labels = test_labels.cuda()\n\n        test_outputs = model(test_inputs)\n        _, predicted = torch.max(test_outputs.data, 1)\n        total += test_labels.size(0)\n        correct += (predicted == test_labels).sum().item()\n\n        all_labels.extend(test_labels.cpu().numpy())\n        all_preds.extend(predicted.cpu().numpy())\n\n        pbar.update()\n\ntest_accuracy = correct / total\nprint(f\"Test Accuracy: {test_accuracy}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:38:32.510782Z","iopub.execute_input":"2023-12-24T00:38:32.511542Z","iopub.status.idle":"2023-12-24T00:38:42.808238Z","shell.execute_reply.started":"2023-12-24T00:38:32.511504Z","shell.execute_reply":"2023-12-24T00:38:42.807277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retraining the model on the validation data\n\ntest_train_epochs = 10\n\nfor epoch in range(test_train_epochs):\n    model.train()\n    running_loss = 0.0\n    total_correct_train = 0\n    total_train = 0\n\n    with tqdm(total=len(test_dataloader), desc=f'Training on test data - Epoch {epoch + 1}/{test_train_epochs}', unit='batch') as pbar:\n        for batch_idx, (inputs, labels) in enumerate(test_dataloader):\n            if torch.cuda.is_available():\n                inputs = inputs.cuda()\n                labels = labels.cuda()\n\n            optimizer.zero_grad()\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            torch.cuda.empty_cache()\n            running_loss += loss.item()\n\n            _, predicted_train = torch.max(outputs.data, 1)\n            total_correct_train += (predicted_train == labels).sum().item()\n            total_train += labels.size(0)\n\n            pbar.set_postfix(loss=loss.item(), acc=total_correct_train / total_train)\n            pbar.update()\n            \n    avg_train_loss = running_loss / len(train_dataloader)\n    train_accuracy = total_correct_train / total_train","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:38:42.809871Z","iopub.execute_input":"2023-12-24T00:38:42.810206Z","iopub.status.idle":"2023-12-24T00:40:51.342249Z","shell.execute_reply.started":"2023-12-24T00:38:42.810178Z","shell.execute_reply":"2023-12-24T00:40:51.341295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting Training and Validation Loss\n\nplt.subplot(1, 2, 1)\nplt.plot(train_loss_history, label='Training Loss')\nplt.plot(val_loss_history, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training and Validation Loss Over Epochs')\nplt.legend()\n\nplt.savefig(\"loss curve.png\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:40:56.155665Z","iopub.execute_input":"2023-12-24T00:40:56.156042Z","iopub.status.idle":"2023-12-24T00:40:56.630529Z","shell.execute_reply.started":"2023-12-24T00:40:56.156002Z","shell.execute_reply":"2023-12-24T00:40:56.629686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting Training and Validation Accuracy\nplt.subplot(1, 2, 2)\nplt.plot(train_accuracy_history, label='Training Accuracy')\nplt.plot(val_accuracy_history, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Training and Validation Accuracy Over Epochs')\nplt.legend()\n\nplt.tight_layout()\n\nplt.savefig(\"accuracy curve.png\")\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:40:56.774897Z","iopub.execute_input":"2023-12-24T00:40:56.775171Z","iopub.status.idle":"2023-12-24T00:40:57.371520Z","shell.execute_reply.started":"2023-12-24T00:40:56.775147Z","shell.execute_reply":"2023-12-24T00:40:57.370654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the confusion matrix\nconf_matrix = confusion_matrix(all_labels, all_preds)\n\n# Plot the confusion matrix\nplt.figure(figsize=(20, 20))\nsns.heatmap(conf_matrix, annot=True, fmt=\"d\", cmap=\"Blues\", xticklabels=list(labels_dict.values()), yticklabels=list(labels_dict.values()))\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted Labels\")\nplt.ylabel(\"True Labels\")\n\nplt.savefig(\"confusion_matrix.png\")\n\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:41:23.436344Z","iopub.execute_input":"2023-12-24T00:41:23.436769Z","iopub.status.idle":"2023-12-24T00:41:36.639795Z","shell.execute_reply.started":"2023-12-24T00:41:23.436734Z","shell.execute_reply":"2023-12-24T00:41:36.638865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_directory = '/kaggle/input/arabic-letters-classification/Final_Arabic_Alpha_dataset/Final_Arabic_Alpha_dataset/test/'\ncsv_file_path = 'predictions.csv'\n\nwith open(csv_file_path, 'w', newline='') as csvfile:\n    csv_writer = csv.writer(csvfile)\n\n    csv_writer.writerow(['ID', 'Label'])\n\n    for filename in sorted(os.listdir(test_directory), key=lambda x: int(x.split('.')[0])):\n        if filename.endswith(\".jpg\") or filename.endswith(\".png\"):\n            image_path = os.path.join(test_directory, filename)\n\n            input_image = Image.open(image_path).convert('L')\n            input_tensor = transform(input_image).unsqueeze(0)\n            input_tensor = input_tensor.cuda()\n            \n\n            with torch.no_grad():\n                output = model(input_tensor)\n\n            _, predicted_class = torch.max(output, 1)\n\n            csv_writer.writerow([int(filename.split('.')[0]), predicted_class.item()])\n\nprint(f\"Predictions saved to: {csv_file_path}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-24T00:41:59.996507Z","iopub.execute_input":"2023-12-24T00:41:59.997395Z","iopub.status.idle":"2023-12-24T00:47:13.628677Z","shell.execute_reply.started":"2023-12-24T00:41:59.997356Z","shell.execute_reply":"2023-12-24T00:47:13.627690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}