{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":74704,"databundleVersionId":8152167,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# HW4\n# Junhyun Kim\n# jk52887\n# kaggle username : Jun_Junhyun_Kim\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-22T03:27:26.535497Z","iopub.execute_input":"2024-04-22T03:27:26.535858Z","iopub.status.idle":"2024-04-22T03:27:26.541091Z","shell.execute_reply.started":"2024-04-22T03:27:26.535827Z","shell.execute_reply":"2024-04-22T03:27:26.540044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\nimport numpy as np\nimport os\nimport random\n\n\nimport torch\nimport torchvision\nimport torchvision.transforms as transform\nfrom torchvision.datasets import ImageFolder\nfrom torch.utils.data import DataLoader, Dataset, ConcatDataset\nimport torch.nn as nn\nimport torchvision.models as models\nfrom torchvision.utils import make_grid\nimport torch.nn.functional as F\n\n\nfrom sklearn.model_selection import train_test_split\n\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:27:28.53856Z","iopub.execute_input":"2024-04-22T03:27:28.539159Z","iopub.status.idle":"2024-04-22T03:27:35.533982Z","shell.execute_reply.started":"2024-04-22T03:27:28.539128Z","shell.execute_reply":"2024-04-22T03:27:35.533064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['dandelion', 'daisy','tulip','sunflower','rose']\nimg_size = 224\ntrain_transform = transform.Compose([\n    transform.Resize((img_size, img_size)),\n    transform.ToTensor(),\n    transform.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\ntrain_dataset = ImageFolder('../input/ece379k-hw4-flower-classification-24spring/train', transform=train_transform)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:27:51.054912Z","iopub.execute_input":"2024-04-22T03:27:51.055714Z","iopub.status.idle":"2024-04-22T03:27:51.296952Z","shell.execute_reply.started":"2024-04-22T03:27:51.055685Z","shell.execute_reply":"2024-04-22T03:27:51.296257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Description\nI added transforms.Normalize() because it is necessary for neural network bacpropagation. Otherwise, gradient vanishing will happen.","metadata":{}},{"cell_type":"code","source":"l = []\nfor i in train_dataset:\n    l.append(labels[i[1]])\nsns.set_style('dark')\nsns.countplot(x=l)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T02:15:02.782977Z","iopub.execute_input":"2024-04-22T02:15:02.784467Z","iopub.status.idle":"2024-04-22T02:15:19.851858Z","shell.execute_reply.started":"2024-04-22T02:15:02.784422Z","shell.execute_reply":"2024-04-22T02:15:19.850574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax=plt.subplots(5,2)\nfig.set_size_inches(15,15)\nfor i in range(5):\n    for j in range (2):\n        l=random.randint(0,len(train_dataset))\n        ax[i,j].imshow(train_dataset[l][0].cpu().numpy().transpose(1, 2, 0))\n        ax[i,j].set_title('Flower: '+labels[train_dataset[l][1]])\n        \nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T02:15:32.667731Z","iopub.execute_input":"2024-04-22T02:15:32.668163Z","iopub.status.idle":"2024-04-22T02:15:37.09356Z","shell.execute_reply.started":"2024-04-22T02:15:32.66813Z","shell.execute_reply":"2024-04-22T02:15:37.091639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_train, my_val = train_test_split(train_dataset, test_size=0.1, shuffle=True, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:28:10.340933Z","iopub.execute_input":"2024-04-22T03:28:10.34131Z","iopub.status.idle":"2024-04-22T03:28:40.969551Z","shell.execute_reply.started":"2024-04-22T03:28:10.341281Z","shell.execute_reply":"2024-04-22T03:28:40.968587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_train_dataloader = DataLoader(my_train, batch_size=128, shuffle=True)\nmy_val_dataloader = DataLoader(my_val, batch_size=128, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:42:42.564458Z","iopub.execute_input":"2024-04-22T03:42:42.5648Z","iopub.status.idle":"2024-04-22T03:42:42.569873Z","shell.execute_reply.started":"2024-04-22T03:42:42.564774Z","shell.execute_reply":"2024-04-22T03:42:42.568895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_Model(torch.nn.Module):\n\n    def __init__(self):\n        super(CNN_Model, self).__init__()\n        \n        self.conv1=torch.nn.Conv2d(3,50,kernel_size=7,stride=2)\n        self.relu1=torch.nn.ReLU()\n        self.pool1=torch.nn.MaxPool2d(2)\n        self.conv2=torch.nn.Conv2d(50,100,kernel_size=7,stride=2)\n        self.relu2=torch.nn.ReLU()\n        self.pool2=torch.nn.MaxPool2d(2)\n        self.linear1 = torch.nn.Linear(14400, 1000)\n        self.activation = torch.nn.Tanh()\n        self.linear2 = torch.nn.Linear(1000, 5)\n        \n\n    def forward(self, x):\n        x=self.conv1(x)\n        x=self.relu1(x)\n        x=self.pool1(x)\n        x=self.conv2(x)\n        x=self.relu2(x)\n        x=self.pool2(x)\n        x=torch.flatten(x,start_dim=1)\n        x = self.linear1(x)\n        x = self.activation(x)\n        x = self.linear2(x)\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:28:50.208572Z","iopub.execute_input":"2024-04-22T03:28:50.208891Z","iopub.status.idle":"2024-04-22T03:28:50.218766Z","shell.execute_reply.started":"2024-04-22T03:28:50.208867Z","shell.execute_reply":"2024-04-22T03:28:50.217781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Description\n\nThis model utilizes convolution layer to identify features of regional part of given image. Each convolution layer has kernel 7 and stride 2. Thus, x,y width of matrix decreases as it feedforwards. However, channels increases from 3 to 50, 50 to 100 in each convolutional layer so that it keeps more features in high level. Activation layers and pooling layers(max pool, kernel size of 2) are put between convolutional layers. Output coming after convolutional layers is 12x12x100 dimesion matrix. This features are flattened so that it can be passed into two linear layers(fully connected layers). Output of linear layers has dimension 5 so that this values can be directly used to compute cross entropy loss.","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\n\nmodel=CNN_Model().to(device)\n\nn_epochs = 30\nlr=0.0001\n\n# Specify loss function\nclassifier = nn.CrossEntropyLoss().to(device)\n\n# Specify optimizer\noptimizer = torch.optim.Adam(model.parameters(), lr=lr)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:44:23.719677Z","iopub.execute_input":"2024-04-22T03:44:23.720361Z","iopub.status.idle":"2024-04-22T03:44:23.887544Z","shell.execute_reply.started":"2024-04-22T03:44:23.720327Z","shell.execute_reply":"2024-04-22T03:44:23.886544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Hyperparameter & Classifier Description\n\nCross-entropy loss was used for classifier. Adam optimizer is used for optimizaition. learning rate is set to 0.0001 after some experiments using different learning rate. I trained 30 epochs for this neural network, and it was enough to typical loss graph.\n","metadata":{}},{"cell_type":"code","source":"# Train & Validation\n\ntrain_loss_history=[]\nval_loss_history=[]\naccuracy_history=[]\n\nfor epoch in range(n_epochs):\n    \n    #train\n    \n    model.train()  # prep model for training\n    train_loss = 0.0\n    \n    for data, target in my_train_dataloader:\n        # Move tensors to the configured device\n        data, target = data.to(device), target.to(device)\n        \n        # Clear the gradients\n        optimizer.zero_grad()\n        # Forward pass: compute predicted outputs\n        output = model(data)\n        # Calculate the loss\n        loss = classifier(output, target)\n        # Backward pass: compute gradient of the loss with respect to model parameters\n        loss.backward()\n        # Perform a single optimization step (parameter update)\n        optimizer.step()\n        # Update running training loss\n        train_loss += loss.item()* data.size(0)\n    \n    # Print training statistics \n    # Calculate average loss over an epoch\n    train_loss = train_loss/len(my_train_dataloader.dataset)\n    print('Epoch: {} \\tTraining Loss: {:.6f}'.format(\n        epoch+1, \n        train_loss\n        ))\n    train_loss_history.append(train_loss)\n    \n    ###############################################################\n    #Validation\n    \n    model.eval()  # Set the model to evaluation mode\n    val_loss = 0.0\n    correct = 0\n    total = 0\n\n    with torch.no_grad():  # Turn off gradients for validation, saves memory and computations\n        for data, targets in my_val_dataloader:\n            data, targets = data.to(device), targets.to(device)\n            outputs = model(data)\n            loss = classifier(outputs, targets)\n            _, predicted = torch.max(outputs, 1)\n            correct += (predicted == targets).sum().item()\n            total += targets.size(0)\n            val_loss+=loss.item()* data.size(0)\n\n    val_loss /= len(my_val_dataloader.dataset)\n    accuracy = 100 * correct / total\n    print(f'Validation Loss: {val_loss:.4f}, Accuracy: {accuracy:.2f}%')\n    \n    val_loss_history.append(val_loss)\n    accuracy_history.append(accuracy)\n    \n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:44:27.979571Z","iopub.execute_input":"2024-04-22T03:44:27.980531Z","iopub.status.idle":"2024-04-22T03:46:18.748963Z","shell.execute_reply.started":"2024-04-22T03:44:27.980497Z","shell.execute_reply":"2024-04-22T03:46:18.747909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots()\n\n# Plot the first line\nax.plot(train_loss_history, label='Train')\n\n# Plot the second line without clearing the first one\nax.plot(val_loss_history, label='Validation')\n\n# Adding labels\nax.set_xlabel('Epochs')\nax.set_ylabel('Loss')\nax.set_title('Loss over Epochs')\n\n# Adding legend\nax.legend()\n\n# plot(train_loss_history)\n# plot(val_loss_history)\n\nfig, ax = plt.subplots()\n\n# Plot the first line\nax.plot(accuracy_history, label='Accuracy')\n\n\n# Adding labels\nax.set_xlabel('Epochs')\nax.set_ylabel('Accuracy')\nax.set_title('Accuracy over Epochs ')\n# plot(accuracy_history)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T03:49:43.239526Z","iopub.execute_input":"2024-04-22T03:49:43.23988Z","iopub.status.idle":"2024-04-22T03:49:43.900796Z","shell.execute_reply.started":"2024-04-22T03:49:43.239852Z","shell.execute_reply":"2024-04-22T03:49:43.899903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Result Discussion\n\nTraining Loss keeps decreasing during training. Validation Loss also decreases at the beginning but at some point increases again. Before this point, the model is underfitting. After this point, it overfits. For accuracy, accuracy increases over epochs.","metadata":{}}]}