{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Assignment 3: CIFAR 10 Dataset ConvNet\n**2 GPUs**<br>\nAbdul-Rahman Abdel-Fattah","metadata":{}},{"cell_type":"code","source":"try:\n    import py7zr\n    import torchmetrics, mlxtend\n    from torchsummary import summary\nexcept:\n    !pip install py7zr\n    !pip install -q torchmetrics -U mlxtend # <- Note: If you're using Google Colab, this may require restarting the runtime\n    !pip install torchsummary\n    \n    import torchmetrics, mlxtend\n    import torchsummary\n    import py7zr","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:08:15.938484Z","iopub.execute_input":"2023-09-19T15:08:15.938957Z","iopub.status.idle":"2023-09-19T15:09:15.49978Z","shell.execute_reply.started":"2023-09-19T15:08:15.938911Z","shell.execute_reply":"2023-09-19T15:09:15.49872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm.auto import tqdm\n\n# Import the visualization library\nimport matplotlib.pyplot as plt\nfrom matplotlib import image\n\n# Import PyTorch\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\n# Torchvision\nfrom torchvision import datasets, transforms\n\n# Import matplotlib for visualization\nimport matplotlib.pyplot as plt\n\nfrom sklearn.preprocessing import LabelEncoder\n\nprint(f\"PyTorch version: {torch.__version__}\")\ndevice = \"cuda\" if torch.cuda.is_available() else None\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:09:15.502056Z","iopub.execute_input":"2023-09-19T15:09:15.502438Z","iopub.status.idle":"2023-09-19T15:09:15.873471Z","shell.execute_reply.started":"2023-09-19T15:09:15.502404Z","shell.execute_reply":"2023-09-19T15:09:15.872416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the Data","metadata":{}},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:09:15.875279Z","iopub.execute_input":"2023-09-19T15:09:15.876014Z","iopub.status.idle":"2023-09-19T15:09:15.884969Z","shell.execute_reply.started":"2023-09-19T15:09:15.875977Z","shell.execute_reply":"2023-09-19T15:09:15.883639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_dir = os.path.join(\"/kaggle\", \"input\", \"cifar-10/\")\noutput_dir = os.path.join(\"/kaggle\", \"output\", \"cifar-10/\")\n\n# Create the output directory if it doesn't exist\nos.makedirs(output_dir, exist_ok=True)\n\ninput_dir, output_dir","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:09:15.889374Z","iopub.execute_input":"2023-09-19T15:09:15.889676Z","iopub.status.idle":"2023-09-19T15:09:15.902421Z","shell.execute_reply.started":"2023-09-19T15:09:15.889652Z","shell.execute_reply":"2023-09-19T15:09:15.901401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_files = [file for file in os.listdir(input_dir) if file[-3:] == \".7z\"]\n\nfor file in zip_files:\n    print(f\"Extracting the file {file}\")\n    \n    with py7zr.SevenZipFile(os.path.join(input_dir, file), mode='r') as z:\n        z.extractall(path=output_dir)\n    \n    print(f\"Extraction Complete\")","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:09:15.903642Z","iopub.execute_input":"2023-09-19T15:09:15.903911Z","iopub.status.idle":"2023-09-19T15:30:25.327914Z","shell.execute_reply.started":"2023-09-19T15:09:15.903888Z","shell.execute_reply":"2023-09-19T15:30:25.326823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the Training Data","metadata":{}},{"cell_type":"code","source":"train_labels = pd.read_csv(input_dir + \"trainLabels.csv\", index_col=[\"id\"]).squeeze()\nlabel_encoder = LabelEncoder().fit(train_labels)\n\nclasses = label_encoder.classes_\n\nprint('Classes in CIFAR10')\nprint(classes)\n\nprint(\"\\nThe number of classes is\")\nprint(len(classes))\n\nprint(\"\\nNumber of training data records\")\nprint(train_labels.shape[0])\n\nprint(\"\\nThe first 5 labels\")\nprint(train_labels.head(5))","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:25.329505Z","iopub.execute_input":"2023-09-19T15:30:25.330118Z","iopub.status.idle":"2023-09-19T15:30:25.41166Z","shell.execute_reply.started":"2023-09-19T15:30:25.330084Z","shell.execute_reply":"2023-09-19T15:30:25.41051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = []\nfor i in range(1, len(train_labels) + 1):\n    img = image.imread('/kaggle/output/cifar-10/train/' + str(i) + \".png\")\n    train_data.append([img, label_encoder.transform([train_labels[i]])[0]])\n    \nlen(train_data)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:25.416799Z","iopub.execute_input":"2023-09-19T15:30:25.419311Z","iopub.status.idle":"2023-09-19T15:30:48.178016Z","shell.execute_reply.started":"2023-09-19T15:30:25.41927Z","shell.execute_reply":"2023-09-19T15:30:48.176866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_encoder.inverse_transform([train_data[0][1]]), train_data[0][0].shape","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:48.179704Z","iopub.execute_input":"2023-09-19T15:30:48.180118Z","iopub.status.idle":"2023-09-19T15:30:48.195617Z","shell.execute_reply.started":"2023-09-19T15:30:48.180081Z","shell.execute_reply":"2023-09-19T15:30:48.194375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <a id=\"visualize\">Visualizing Sample Images</a>","metadata":{}},{"cell_type":"code","source":"idx = np.random.randint(50000)\n\nplt.imshow(train_data[idx][0])\nplt.title(label_encoder.inverse_transform([train_data[idx][1]])[0])\nplt.axis(False)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:48.198467Z","iopub.execute_input":"2023-09-19T15:30:48.199732Z","iopub.status.idle":"2023-09-19T15:30:48.440318Z","shell.execute_reply.started":"2023-09-19T15:30:48.199696Z","shell.execute_reply":"2023-09-19T15:30:48.43922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\n \nnum_rows = 3\nnum_cols = 8\n\nfor idx in range(num_rows * num_cols):\n    idx_to_show = np.random.choice(len(train_labels))\n    img    = train_data[idx_to_show][0]\n    label  = label_encoder.inverse_transform([train_data[idx_to_show][1]])[0]\n        \n    ax = plt.subplot(num_rows, num_cols, idx + 1)\n    plt.imshow(img)\n    plt.title(label)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:48.445705Z","iopub.execute_input":"2023-09-19T15:30:48.44802Z","iopub.status.idle":"2023-09-19T15:30:50.182243Z","shell.execute_reply.started":"2023-09-19T15:30:48.447988Z","shell.execute_reply":"2023-09-19T15:30:50.181228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creation of and Training The Model","metadata":{}},{"cell_type":"code","source":"class Configs:\n    batch_size = 32,\n    shuffle    = True,\n    epochs     = 100\n    \nconfigs = Configs()","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:50.188246Z","iopub.execute_input":"2023-09-19T15:30:50.188623Z","iopub.status.idle":"2023-09-19T15:30:50.193936Z","shell.execute_reply.started":"2023-09-19T15:30:50.188591Z","shell.execute_reply":"2023-09-19T15:30:50.193077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## <a id=\"torch\">Pytorch Model</a>","metadata":{}},{"cell_type":"markdown","source":"### Creation","metadata":{}},{"cell_type":"code","source":"class CNN_Model(nn.Module):\n    def __init__(self, input_shape: int = 3, output_shape: int = 10):\n        super().__init__()\n\n        self.conv1 = nn.Conv2d(3, 32, 5)\n        self.conv2 = nn.Conv2d(32, 64, 5)\n        self.pool = nn.MaxPool2d(2, 2)\n\n        self.fc1 = nn.Linear(64 * 5 * 5, 120)\n        self.fc2 = nn.Linear(120, 84)\n        self.fc3 = nn.Linear(84, 10)\n\n    def forward(self, x):\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = torch.flatten(x, 1) # flatten all dimensions except batch\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = self.fc3(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:50.195648Z","iopub.execute_input":"2023-09-19T15:30:50.19633Z","iopub.status.idle":"2023-09-19T15:30:50.210204Z","shell.execute_reply.started":"2023-09-19T15:30:50.196273Z","shell.execute_reply":"2023-09-19T15:30:50.209104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN_Model()\n\n# Setup loss and optimizer\nloss_fn = nn.CrossEntropyLoss().to(device)\noptimizer = torch.optim.SGD(params=model.parameters(), lr=0.001, momentum=0.9)\naccuracy = torchmetrics.Accuracy(task=\"multiclass\", num_classes=10).to(device)\n\n# Running on multiple GPUs\nmodel = nn.DataParallel(model, device_ids=[0, 1])\n\nmodel.to(device)\n#summary(model, (3, 32, 32))","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:50.211812Z","iopub.execute_input":"2023-09-19T15:30:50.212596Z","iopub.status.idle":"2023-09-19T15:30:56.171641Z","shell.execute_reply.started":"2023-09-19T15:30:50.212559Z","shell.execute_reply":"2023-09-19T15:30:56.170651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preparation","metadata":{}},{"cell_type":"code","source":"train_dataloader = DataLoader(\n    train_data, \n    batch_size   = configs.batch_size[0],\n    shuffle      = configs.shuffle,\n)\n\nprint(f\"Length of train dataloader: {len(train_dataloader)} batches of {configs.batch_size[0]}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:56.173177Z","iopub.execute_input":"2023-09-19T15:30:56.173811Z","iopub.status.idle":"2023-09-19T15:30:56.180282Z","shell.execute_reply.started":"2023-09-19T15:30:56.173766Z","shell.execute_reply":"2023-09-19T15:30:56.179367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training the Model","metadata":{}},{"cell_type":"code","source":"model.train()\nmodel.to(device)\n\nfor epoch in tqdm(range(configs.epochs)):\n    print(f\"Epoch: {epoch+1}\\n---------\")\n\n    train_loss, train_acc = 0, 0\n    \n    for batch, (X, y) in enumerate(train_dataloader):\n        # Send data to GPU\n        X, y = X.reshape(-1, 3, 32, 32).to(device), y.to(device)\n        \n        # Optimizer zero grad\n        optimizer.zero_grad()\n        \n        #  Forward pass\n        y_logits = model(X)\n        y_preds  = torch.softmax(y_logits, dim=1)\n        #  Calculate loss\n        loss = loss_fn(y_logits, y) \n        #  Loss backward\n        loss.backward()\n        optimizer.step()\n        \n        train_loss += loss\n        train_acc += accuracy(y, y_preds.argmax(dim=1)) # Go from logits -> pred labels\n        \n    # Calculate loss and accuracy per epoch and print out what's happening\n    train_loss /= len(train_dataloader)\n    train_acc /= len(train_dataloader)\n    print(f\"Train loss: {train_loss:.5f} | Train accuracy: {train_acc:.2f}%\")","metadata":{"execution":{"iopub.status.busy":"2023-09-19T15:30:56.181748Z","iopub.execute_input":"2023-09-19T15:30:56.182574Z","iopub.status.idle":"2023-09-19T15:50:59.843394Z","shell.execute_reply.started":"2023-09-19T15:30:56.18247Z","shell.execute_reply":"2023-09-19T15:50:59.842381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction on Test Images","metadata":{}},{"cell_type":"code","source":"idxs = np.random.choice(1000, 24)\n\ntest_data = []\nfor i in idxs:\n    img = image.imread('/kaggle/output/cifar-10/test/' + str(i) + \".png\")\n    test_data.append(img)\n    \ntest_data = np.array(test_data)\ntest_data.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:29:40.605032Z","iopub.execute_input":"2023-09-19T16:29:40.605461Z","iopub.status.idle":"2023-09-19T16:29:40.631213Z","shell.execute_reply.started":"2023-09-19T16:29:40.605425Z","shell.execute_reply":"2023-09-19T16:29:40.629876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.eval()\ntest_labels = model(torch.tensor(test_data.reshape(-1, 3, 32, 32)).to(device)).argmax(dim=1)\ntest_labels = label_encoder.inverse_transform(np.array(test_labels.cpu()))\n\ntest_labels","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:29:43.636942Z","iopub.execute_input":"2023-09-19T16:29:43.637331Z","iopub.status.idle":"2023-09-19T16:29:43.660233Z","shell.execute_reply.started":"2023-09-19T16:29:43.637297Z","shell.execute_reply":"2023-09-19T16:29:43.659214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = np.random.randint(18)\n\nplt.imshow(test_data[idx])\nplt.title(test_labels[idx])\nplt.axis(False)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:29:46.286297Z","iopub.execute_input":"2023-09-19T16:29:46.287776Z","iopub.status.idle":"2023-09-19T16:29:46.485223Z","shell.execute_reply.started":"2023-09-19T16:29:46.287708Z","shell.execute_reply":"2023-09-19T16:29:46.484221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15, 5))\n \nnum_rows = 3\nnum_cols = 8\n\nfor idx in range(num_rows * num_cols):\n    img         = test_data[idx]\n    label       = test_labels[idx]\n    \n    ax = plt.subplot(num_rows, num_cols, idx + 1)\n    plt.imshow(img)\n    plt.title(label)\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:29:49.787603Z","iopub.execute_input":"2023-09-19T16:29:49.788813Z","iopub.status.idle":"2023-09-19T16:29:51.889238Z","shell.execute_reply.started":"2023-09-19T16:29:49.788772Z","shell.execute_reply":"2023-09-19T16:29:51.888322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# putting the model in evaluation mode\nmodel.eval()","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:29:57.776123Z","iopub.execute_input":"2023-09-19T16:29:57.777085Z","iopub.status.idle":"2023-09-19T16:29:57.784369Z","shell.execute_reply.started":"2023-09-19T16:29:57.777041Z","shell.execute_reply":"2023-09-19T16:29:57.783214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Emptying some RAM storage","metadata":{}},{"cell_type":"code","source":"%whos\ndel(train_dataloader)\ndel(train_data)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:30:00.793025Z","iopub.execute_input":"2023-09-19T16:30:00.794662Z","iopub.status.idle":"2023-09-19T16:30:00.960515Z","shell.execute_reply.started":"2023-09-19T16:30:00.794613Z","shell.execute_reply":"2023-09-19T16:30:00.959581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading the test dataset","metadata":{}},{"cell_type":"code","source":"no_of_test_images = len(os.listdir(\"/kaggle/output/cifar-10/test\"))\nmodel.eval()\ntest_data = []\n\nfor i in range(1, no_of_test_images + 1):\n    img = image.imread('/kaggle/output/cifar-10/test/' + str(i) + \".png\")\n    test_data.append([img, i]) ","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:30:06.113499Z","iopub.execute_input":"2023-09-19T16:30:06.113893Z","iopub.status.idle":"2023-09-19T16:32:04.897459Z","shell.execute_reply.started":"2023-09-19T16:30:06.11386Z","shell.execute_reply":"2023-09-19T16:32:04.896163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(\n    test_data, \n    batch_size   = configs.batch_size[0],\n    shuffle      = False,\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:32:11.512931Z","iopub.execute_input":"2023-09-19T16:32:11.513313Z","iopub.status.idle":"2023-09-19T16:32:11.522279Z","shell.execute_reply.started":"2023-09-19T16:32:11.513274Z","shell.execute_reply":"2023-09-19T16:32:11.521382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_data)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:32:14.902833Z","iopub.execute_input":"2023-09-19T16:32:14.903201Z","iopub.status.idle":"2023-09-19T16:32:14.909824Z","shell.execute_reply.started":"2023-09-19T16:32:14.903169Z","shell.execute_reply":"2023-09-19T16:32:14.908655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predicting the results","metadata":{}},{"cell_type":"code","source":"ids = []\nlabels = []\n\nfor (X, y) in tqdm(test_dataloader):\n    X = X.reshape(-1, 3, 32, 32)\n    y_logits = model(X)\n    y_preds  = torch.softmax(y_logits, dim=1).argmax(dim=1).cpu()\n    labels += label_encoder.inverse_transform(y_preds).tolist()\n    ids += y.squeeze().tolist()\n\n","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:32:17.021102Z","iopub.execute_input":"2023-09-19T16:32:17.021531Z","iopub.status.idle":"2023-09-19T16:32:59.947389Z","shell.execute_reply.started":"2023-09-19T16:32:17.021497Z","shell.execute_reply":"2023-09-19T16:32:59.946315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(ids), len(labels), ids[1], labels[1]","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:33:06.269866Z","iopub.execute_input":"2023-09-19T16:33:06.270899Z","iopub.status.idle":"2023-09-19T16:33:06.280308Z","shell.execute_reply.started":"2023-09-19T16:33:06.270851Z","shell.execute_reply":"2023-09-19T16:33:06.279061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a DataFrame for the submission\nsubmission_df = pd.DataFrame({\n    'id': ids,\n    'label': labels\n})\n\n# Save the submission file with class labels\nsubmission_df.to_csv('/kaggle/working/submission3.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-19T16:33:08.787111Z","iopub.execute_input":"2023-09-19T16:33:08.788275Z","iopub.status.idle":"2023-09-19T16:33:09.65465Z","shell.execute_reply.started":"2023-09-19T16:33:08.788232Z","shell.execute_reply":"2023-09-19T16:33:09.653619Z"},"trusted":true},"execution_count":null,"outputs":[]}]}