{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Assignment 3: CIFAR 10 Dataset ConvNet\nAbdul-Rahman Abdel-Fattah","metadata":{}},{"cell_type":"code","source":"try:\n    import py7zr\n    import torchmetrics, mlxtend\n    from torchsummary import summary\nexcept:\n    !pip install py7zr\n    !pip install -q torchmetrics -U mlxtend # <- Note: If you're using Google Colab, this may require restarting the runtime\n    !pip install torchsummary\n    \n    import torchmetrics, mlxtend\n    import torchsummary\n    import py7zr","metadata":{"execution":{"iopub.status.busy":"2023-09-16T21:01:31.838739Z","iopub.execute_input":"2023-09-16T21:01:31.839594Z","iopub.status.idle":"2023-09-16T21:01:31.849365Z","shell.execute_reply.started":"2023-09-16T21:01:31.839551Z","shell.execute_reply":"2023-09-16T21:01:31.848377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\n\n# Import the visualization library\nimport matplotlib.pyplot as plt\nfrom matplotlib import image\n\n# Import PyTorch\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\n# imports the torch_xla package\nimport torch_xla\nimport torch_xla.core.xla_model as xm\n\n# Import matplotlib for visualization\nimport matplotlib.pyplot as plt\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.preprocessing import LabelEncoder\n\nprint(f\"PyTorch version: {torch.__version__}\")\ndevice = xm.xla_device()\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-09-16T21:01:32.620046Z","iopub.execute_input":"2023-09-16T21:01:32.620467Z","iopub.status.idle":"2023-09-16T21:01:32.631021Z","shell.execute_reply.started":"2023-09-16T21:01:32.620437Z","shell.execute_reply":"2023-09-16T21:01:32.630021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the Data","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-09-16T21:01:41.032016Z","iopub.execute_input":"2023-09-16T21:01:41.032443Z","iopub.status.idle":"2023-09-16T21:01:41.038891Z","shell.execute_reply.started":"2023-09-16T21:01:41.032411Z","shell.execute_reply":"2023-09-16T21:01:41.037867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_dir = os.path.join(\"/kaggle\", \"input\", \"cifar-10/\")\noutput_dir = os.path.join(\"/kaggle\", \"output\", \"cifar-10/\")\n\n# Create the output directory if it doesn't exist\nos.makedirs(output_dir, exist_ok=True)\n\ninput_dir, output_dir","metadata":{"execution":{"iopub.status.busy":"2023-09-16T21:01:50.68469Z","iopub.execute_input":"2023-09-16T21:01:50.685061Z","iopub.status.idle":"2023-09-16T21:01:50.692849Z","shell.execute_reply.started":"2023-09-16T21:01:50.685032Z","shell.execute_reply":"2023-09-16T21:01:50.691972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_files = [file for file in os.listdir(input_dir) if file[-3:] == \".7z\"]\n\nfor file in zip_files:\n    print(f\"Extracting the file {file}\")\n    \n    with py7zr.SevenZipFile(os.path.join(input_dir, file), mode='r') as z:\n        z.extractall(path=output_dir)\n    \n    print(f\"Extraction Complete\")","metadata":{"execution":{"iopub.status.busy":"2023-09-16T21:01:51.41107Z","iopub.execute_input":"2023-09-16T21:01:51.411495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv(input_dir + \"trainLabels.csv\", index_col=[\"id\"]).squeeze()\nlabel_encoder = LabelEncoder().fit(train_labels)\n\nclasses = label_encoder.classes_\n\nprint('Classes in CIFAR10')\nprint(classes)\n\nprint(\"\\nThe number of classes is\")\nprint(len(classes))\n\nprint(\"\\nNumber of Labels\")\nprint(train_labels.shape[0])\n\nprint(\"\\nThe first 5 labels\")\nprint(train_labels.head(5))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = []\nfor i in range(1, len(train_labels) + 1):\n    img = image.imread('/kaggle/output/cifar-10/train/' + str(i) + \".png\")\n    train_data.append([img, label_encoder.transform([train_labels[i]])[0]])\n    \nlen(train_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder.inverse_transform([train_data[0][1]]), train_data[0][0].shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id=\"visualize\">Visualizing Sample Images</a>","metadata":{}},{"cell_type":"code","source":"idx = np.random.randint(50000)\n\nplt.imshow(train_data[idx][0])\nplt.title(label_encoder.inverse_transform([train_data[idx][1]])[0])\nplt.axis(False)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T19:14:43.248313Z","iopub.execute_input":"2023-09-16T19:14:43.248724Z","iopub.status.idle":"2023-09-16T19:14:43.482421Z","shell.execute_reply.started":"2023-09-16T19:14:43.24869Z","shell.execute_reply":"2023-09-16T19:14:43.481488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\n \nnum_rows = 3\nnum_cols = 8\n\nfor idx in range(num_rows * num_cols):\n    idx_to_show = np.random.choice(len(train_labels))\n    img    = train_data[idx_to_show][0]\n    label  = label_encoder.inverse_transform([train_data[idx_to_show][1]])[0]\n        \n    ax = plt.subplot(num_rows, num_cols, idx + 1)\n    plt.imshow(img)\n    plt.title(label)\n    plt.axis(\"off\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creation of and Training The Model","metadata":{}},{"cell_type":"code","source":"class Configs:\n    batch_size = 32,\n    shuffle    = True,\n    epochs     = 50\n    \nconfigs = Configs()","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:14:53.446504Z","iopub.execute_input":"2023-09-16T20:14:53.446891Z","iopub.status.idle":"2023-09-16T20:14:53.452518Z","shell.execute_reply.started":"2023-09-16T20:14:53.446861Z","shell.execute_reply":"2023-09-16T20:14:53.451245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id=\"torch\">Pytorch Model</a>","metadata":{}},{"cell_type":"markdown","source":"### Creation","metadata":{}},{"cell_type":"code","source":"class CNN_Model(nn.Module):\n    def __init__(self, input_shape: int = 3, output_shape: int = 10):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(3, 6, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n        self.conv2 = nn.Conv2d(6, 16, 5)\n        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:32:59.12007Z","iopub.execute_input":"2023-09-16T20:32:59.120461Z","iopub.status.idle":"2023-09-16T20:32:59.131554Z","shell.execute_reply.started":"2023-09-16T20:32:59.12043Z","shell.execute_reply":"2023-09-16T20:32:59.130572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN_Model()\n\n# Setup loss and optimizer\nloss_fn = nn.CrossEntropyLoss().to(device)\noptimizer = torch.optim.SGD(params=model.parameters(), lr=0.001, momentum=0.9)\naccuracy = torchmetrics.Accuracy(task=\"multiclass\", num_classes=10).to(device)\n\n# Running on multiple GPUs\nmodel = nn.DataParallel(model, device_ids=[0, 1])\n\nmodel.to(device)\nsummary(model, (3, 32, 32))","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:33:00.540663Z","iopub.execute_input":"2023-09-16T20:33:00.5417Z","iopub.status.idle":"2023-09-16T20:33:00.586383Z","shell.execute_reply.started":"2023-09-16T20:33:00.54165Z","shell.execute_reply":"2023-09-16T20:33:00.584764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preparation","metadata":{}},{"cell_type":"code","source":"train_dataloader = DataLoader(\n    train_data, \n    batch_size   = configs.batch_size[0],\n    shuffle      = configs.shuffle,\n)\n\nprint(f\"Length of train dataloader: {len(train_dataloader)} batches of {configs.batch_size[0]}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:37:12.855624Z","iopub.execute_input":"2023-09-16T20:37:12.856004Z","iopub.status.idle":"2023-09-16T20:37:12.86302Z","shell.execute_reply.started":"2023-09-16T20:37:12.855975Z","shell.execute_reply":"2023-09-16T20:37:12.861949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the Model","metadata":{}},{"cell_type":"code","source":"model.train()\nmodel.to(device)\n\nfor epoch in tqdm(range(configs.epochs)):\n    print(f\"Epoch: {epoch+1}\\n---------\")\n\n    train_loss, train_acc = 0, 0\n    \n    for batch, (X, y) in enumerate(train_dataloader):\n        # Send data to GPU\n        X, y = X.reshape(-1, 3, 32, 32).to(device), y.to(device)\n        \n        # Optimizer zero grad\n        optimizer.zero_grad()\n        \n        #  Forward pass\n        y_logits = model(X)\n        y_preds  = torch.softmax(y_logits, dim=1)\n        #  Calculate loss\n        loss = loss_fn(y_logits, y) \n        #  Loss backward\n        loss.backward()\n        optimizer.step()\n        \n        train_loss += loss\n        train_acc += accuracy(y, y_preds.argmax(dim=1)) # Go from logits -> pred labels\n        \n    # Calculate loss and accuracy per epoch and print out what's happening\n    train_loss /= len(train_dataloader)\n    train_acc /= len(train_dataloader)\n    print(f\"Train loss: {train_loss:.5f} | Train accuracy: {train_acc:.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:49:56.041935Z","iopub.execute_input":"2023-09-16T20:49:56.042308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction on Test Images","metadata":{}},{"cell_type":"code","source":"idxs = np.random.choice(1000, 18)\n\ntest_data = []\nfor i in idxs:\n    img = image.imread('/kaggle/output/cifar-10/test/' + str(i) + \".png\")\n    test_data.append(img)\n    \ntest_data = np.array(test_data)\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:49:36.438589Z","iopub.execute_input":"2023-09-16T20:49:36.439768Z","iopub.status.idle":"2023-09-16T20:49:36.470402Z","shell.execute_reply.started":"2023-09-16T20:49:36.439729Z","shell.execute_reply":"2023-09-16T20:49:36.469225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\ntest_labels = model(torch.tensor(test_data.reshape(-1, 3, 32, 32)).to(device)).argmax(dim=1)\ntest_labels = label_encoder.inverse_transform(np.array(test_labels.cpu()))\n\ntest_labels","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:49:37.13907Z","iopub.execute_input":"2023-09-16T20:49:37.139452Z","iopub.status.idle":"2023-09-16T20:49:37.163532Z","shell.execute_reply.started":"2023-09-16T20:49:37.13942Z","shell.execute_reply":"2023-09-16T20:49:37.162503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = np.random.randint(18)\n\nplt.imshow(test_data[idx])\nplt.title(test_labels[idx])\nplt.axis(False)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:49:37.408381Z","iopub.execute_input":"2023-09-16T20:49:37.409465Z","iopub.status.idle":"2023-09-16T20:49:37.594629Z","shell.execute_reply.started":"2023-09-16T20:49:37.409434Z","shell.execute_reply":"2023-09-16T20:49:37.593509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\n \nnum_rows = 3\nnum_cols = 8\n\nfor idx in range(num_rows * num_cols):\n    idx_to_show = np.random.choice(18)\n    img         = test_data[idx_to_show]\n    label       = test_labels[idx_to_show]\n    \n    ax = plt.subplot(num_rows, num_cols, idx + 1)\n    plt.imshow(img)\n    plt.title(label)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-09-16T20:49:47.999453Z","iopub.execute_input":"2023-09-16T20:49:47.999899Z","iopub.status.idle":"2023-09-16T20:49:49.663591Z","shell.execute_reply.started":"2023-09-16T20:49:47.999868Z","shell.execute_reply":"2023-09-16T20:49:49.662483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"model.zero_grad()\noptimizer.zero_grad()","metadata":{"execution":{"iopub.status.busy":"2023-09-16T18:28:51.306317Z","iopub.execute_input":"2023-09-16T18:28:51.307069Z","iopub.status.idle":"2023-09-16T18:28:51.312551Z","shell.execute_reply.started":"2023-09-16T18:28:51.307035Z","shell.execute_reply":"2023-09-16T18:28:51.311395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading the test dataset","metadata":{}},{"cell_type":"code","source":"no_of_test_images = len(os.listdir(\"/kaggle/output/cifar-10/test\"))\nmodel.eval()\n\nids    = []\nlabels = []\n\nfor i in range(1, no_of_test_images + 1, 50):\n    j = i\n    test_data = []\n    \n    while j < min(no_of_test_images, i+50):\n        img = image.imread('/kaggle/output/cifar-10/test/' + str(j) + \".png\")\n        \n        test_data.append(img)     \n        j += 1\n    \n    test_data = np.array(test_data)\n    \n    y_logits = model(test_data)\n    y_preds  = torch.softmax(y_logits, dim=1).argmax(dim=1)\n    labels.append(label_encoder.inverse_transform([y_preds]))\n        \n","metadata":{"execution":{"iopub.status.busy":"2023-09-16T18:49:21.09194Z","iopub.execute_input":"2023-09-16T18:49:21.092325Z","iopub.status.idle":"2023-09-16T18:49:21.44444Z","shell.execute_reply.started":"2023-09-16T18:49:21.092284Z","shell.execute_reply":"2023-09-16T18:49:21.442767Z"},"trusted":true},"execution_count":null,"outputs":[]}]}