{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# for dirname, _, filenames in os.walk('/kaggle/working'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-22T03:10:18.732546Z","iopub.execute_input":"2023-06-22T03:10:18.733017Z","iopub.status.idle":"2023-06-22T03:10:18.754368Z","shell.execute_reply.started":"2023-06-22T03:10:18.732979Z","shell.execute_reply":"2023-06-22T03:10:18.753340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:10:18.756514Z","iopub.execute_input":"2023-06-22T03:10:18.757091Z","iopub.status.idle":"2023-06-22T03:10:33.436336Z","shell.execute_reply.started":"2023-06-22T03:10:18.757057Z","shell.execute_reply":"2023-06-22T03:10:33.435235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install wandb","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:10:33.439009Z","iopub.execute_input":"2023-06-22T03:10:33.439390Z","iopub.status.idle":"2023-06-22T03:10:44.256232Z","shell.execute_reply.started":"2023-06-22T03:10:33.439350Z","shell.execute_reply":"2023-06-22T03:10:44.255111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import DataLoader, Dataset, ConcatDataset, SubsetRandomSampler\nfrom torchvision.datasets import MNIST\nfrom torchvision import datasets, transforms\nfrom torchvision.models import resnet18\nimport torchvision\nimport torch.nn as nn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nimport cv2\nimport os\nfrom tqdm.notebook import trange, tqdm\nimport albumentations\nfrom timeit import default_timer as timer\nimport glob\nimport py7zr\nfrom sklearn.model_selection import train_test_split\nimport wandb\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:38:45.293208Z","iopub.execute_input":"2023-06-22T03:38:45.293574Z","iopub.status.idle":"2023-06-22T03:38:45.382630Z","shell.execute_reply.started":"2023-06-22T03:38:45.293544Z","shell.execute_reply":"2023-06-22T03:38:45.381724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_path = '/kaggle/working/'\nif not os.path.exists(temp_path):\n    os.mkdir(temp_path)\ntrain_file_path = '../input/cifar-10/train.7z'\narchive = py7zr.SevenZipFile(train_file_path, mode='r')\narchive.extractall(path=temp_path)\narchive.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:10:49.520168Z","iopub.execute_input":"2023-06-22T03:10:49.520593Z","iopub.status.idle":"2023-06-22T03:11:48.065470Z","shell.execute_reply.started":"2023-06-22T03:10:49.520560Z","shell.execute_reply":"2023-06-22T03:11:48.064437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_path = '/kaggle/working/'\nif not os.path.exists(temp_path):\n    os.mkdir(temp_path)\ntest_label_file = '../input/cifar-10/test.7z'\narchive = py7zr.SevenZipFile(test_label_file, mode='r')\narchive.extractall(path=temp_path)\narchive.close()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:11:48.066746Z","iopub.execute_input":"2023-06-22T03:11:48.067101Z","iopub.status.idle":"2023-06-22T03:31:31.455815Z","shell.execute_reply.started":"2023-06-22T03:11:48.067065Z","shell.execute_reply":"2023-06-22T03:31:31.454774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import wandb\n# wandb.login()\n# from kaggle_secrets import UserSecretsClient\n# user_secrets = UserSecretsClient()\n# secret_value_0 = user_secrets.get_secret(\"wandb_api\")\n# wandb.login(key= secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.457467Z","iopub.execute_input":"2023-06-22T03:31:31.457884Z","iopub.status.idle":"2023-06-22T03:31:31.463036Z","shell.execute_reply.started":"2023-06-22T03:31:31.457849Z","shell.execute_reply":"2023-06-22T03:31:31.462105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# os.environ['WANDB_API_KEY'] = secret_value_0","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.464565Z","iopub.execute_input":"2023-06-22T03:31:31.465200Z","iopub.status.idle":"2023-06-22T03:31:31.471725Z","shell.execute_reply.started":"2023-06-22T03:31:31.465168Z","shell.execute_reply":"2023-06-22T03:31:31.470765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import wandb\n\n# # start a new wandb run to track this script\n# wandb.init(\n#     # set the wandb project where this run will be logged\n#     project=\"temporal ensembling test\",\n    \n#     # track hyperparameters and run metadata\n#     config={\n#     \"learning_rate\": 0.0001,\n#     \"architecture\": \"Temporal ensembling\",\n#     \"dataset\": \"CIFAR-10\",\n#     \"epochs\": 10,\n#     }\n# )\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.472815Z","iopub.execute_input":"2023-06-22T03:31:31.473262Z","iopub.status.idle":"2023-06-22T03:31:31.481221Z","shell.execute_reply.started":"2023-06-22T03:31:31.473231Z","shell.execute_reply":"2023-06-22T03:31:31.480366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# images = glob.glob('/kaggle/working/train/*.png')\n# len(images)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.482558Z","iopub.execute_input":"2023-06-22T03:31:31.482903Z","iopub.status.idle":"2023-06-22T03:31:31.490363Z","shell.execute_reply.started":"2023-06-22T03:31:31.482871Z","shell.execute_reply":"2023-06-22T03:31:31.489267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labeled_images=glob.glob('/kaggle/working/train/*.png')\nlen(labeled_images)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.491616Z","iopub.execute_input":"2023-06-22T03:31:31.492074Z","iopub.status.idle":"2023-06-22T03:31:31.674086Z","shell.execute_reply.started":"2023-06-22T03:31:31.492041Z","shell.execute_reply":"2023-06-22T03:31:31.672966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images,val_images= train_test_split(labeled_images,test_size=0.33,random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_images= glob.glob('/kaggle/working/test/*.png')\nlen(test_images)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:15.262518Z","iopub.execute_input":"2023-06-22T03:37:15.263182Z","iopub.status.idle":"2023-06-22T03:37:16.300216Z","shell.execute_reply.started":"2023-06-22T03:37:15.263144Z","shell.execute_reply":"2023-06-22T03:37:16.299329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_label = ['frog', 'truck', 'deer', 'automobile', 'bird', 'horse', 'ship', 'cat', 'dog', 'airplane']\n\nlabel_dict = {}  # Create an empty dictionary\n\nfor index, label in enumerate(temp_label):\n    label_dict[label] = index\n\nprint(label_dict)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:08:25.660201Z","iopub.execute_input":"2023-06-22T05:08:25.661325Z","iopub.status.idle":"2023-06-22T05:08:25.667617Z","shell.execute_reply.started":"2023-06-22T05:08:25.661279Z","shell.execute_reply":"2023-06-22T05:08:25.666674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Cifar10(Dataset):\n  def __init__(self,labeled_images,label_files= None,transform=None,label_dict=None):\n    self.transform = transform\n    if label_files !=None:\n        self.labels= pd.read_csv(label_files)\n    self.list_images=labeled_images\n    self.label_dict=label_dict\n\n    \n   \n    print(len(self.list_images))\n  def __len__(self):\n     return len(self.list_images)\n\n  def __getitem__(self, idx):\n    image_path=self.list_images[idx]\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, (224, 224))\n    \n    \n    if self.transform:\n        res = self.transform(image=image)\n        image = res['image'].astype(np.float32)\n    else:\n        image = image.astype(np.float32)\n\n    image = image.transpose(2, 0, 1)\n    \n    \n  \n    label= self.labels.iloc[int(image_path[22:-4])-1]['label']\n    index=label_dict[label]\n\n    \n    sample= (image,index)\n    return sample # image, label","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:30.125177Z","iopub.execute_input":"2023-06-22T03:37:30.126108Z","iopub.status.idle":"2023-06-22T03:37:30.136572Z","shell.execute_reply.started":"2023-06-22T03:37:30.126062Z","shell.execute_reply":"2023-06-22T03:37:30.135196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Cifar10test(Dataset):\n  def __init__(self,test_images,transform=None):\n    self.transform = transform\n    self.test_images= test_images\n    \n   \n    print(len(self.test_images))\n  def __len__(self):\n     return len(self.test_images)\n\n  def __getitem__(self, idx):\n    image_path=self.test_images[idx]\n    image = cv2.imread(image_path)\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n    image = cv2.resize(image, (224, 224))\n    \n    \n    if self.transform:\n        res = self.transform(image=image)\n        image = res['image'].astype(np.float32)\n    else:\n        image = image.astype(np.float32)\n\n    image = image.transpose(2, 0, 1)\n \n    \n \n    \n    sample=image\n    return sample # image, label","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:07:44.621378Z","iopub.execute_input":"2023-06-22T05:07:44.621776Z","iopub.status.idle":"2023-06-22T05:07:44.629942Z","shell.execute_reply.started":"2023-06-22T05:07:44.621739Z","shell.execute_reply":"2023-06-22T05:07:44.628720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(image_size=(224, 224)):\n\n    transforms_train = albumentations.Compose([\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.ImageCompression(quality_lower=99, quality_upper=100),\n        albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n        # albumentations.Resize(image_size, image_size),\n        # albumentations.Cutout(max_h_size=int(image_size * 0.4), max_w_size=int(image_size * 0.4), num_holes=1, p=0.5),\n        albumentations.Normalize()\n    ])\n\n    transforms_val = albumentations.Compose([\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.ImageCompression(quality_lower=99, quality_upper=100),\n        albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n        # albumentations.Resize(image_size, image_size),\n        # albumentations.Cutout(max_h_size=int(image_size * 0.4), max_w_size=int(image_size * 0.4), num_holes=1, p=0.5),\n        albumentations.Normalize()\n    ])\n\n    return transforms_train, transforms_val","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:34.674031Z","iopub.execute_input":"2023-06-22T03:37:34.674400Z","iopub.status.idle":"2023-06-22T03:37:34.682024Z","shell.execute_reply.started":"2023-06-22T03:37:34.674370Z","shell.execute_reply":"2023-06-22T03:37:34.680912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms_train, transforms_val = get_transforms(image_size=(224, 224))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:36.686971Z","iopub.execute_input":"2023-06-22T03:37:36.687321Z","iopub.status.idle":"2023-06-22T03:37:36.699148Z","shell.execute_reply.started":"2023-06-22T03:37:36.687292Z","shell.execute_reply":"2023-06-22T03:37:36.697991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datas=Cifar10(train_images,label_files='/kaggle/input/cifar-10/trainLabels.csv',transform= transforms_train,label_dict=label_dict)\nval_datas=Cifar10(val_images,label_files='/kaggle/input/cifar-10/trainLabels.csv',transform= transforms_val,label_dict=label_dict)\ntest_data= Cifar10test(test_images,transform= None)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:07:48.561950Z","iopub.execute_input":"2023-06-22T05:07:48.562318Z","iopub.status.idle":"2023-06-22T05:07:48.604309Z","shell.execute_reply.started":"2023-06-22T05:07:48.562289Z","shell.execute_reply":"2023-06-22T05:07:48.603232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n    Function for computing the accuracy of the predictions over the entire data_loader\n'''\ndef get_accuracy(model, data_loader, device):\n    correct = 0\n    total = 0\n\n    with torch.no_grad():\n        model.eval()\n        for images, labels in data_loader:\n            images = images.to(device)\n            labels = labels.to(device)\n\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    return 100*(correct/total)\n\n'''\n    Function for plotting training and validation losses\n'''\ndef plot_losses(train_acc, valid_acc, train_loss, valid_loss):\n    # change the style of the plots to seaborn\n    plt.style.use('seaborn')\n\n    train_acc = np.array(train_acc)\n    valid_acc = np.array(valid_acc)\n\n    fig, (ax1, ax2) = plt.subplots(1, 2)\n\n    ax1.plot(train_acc, color=\"blue\", label=\"Train_acc\")\n    ax1.plot(valid_acc, color=\"red\", label=\"Validation_acc\")\n    ax1.set(title=\"Acc over epochs\",\n            xlabel=\"Epoch\",\n            ylabel=\"Acc\")\n    ax1.legend()\n\n    ax2.plot(train_loss, color=\"blue\", label=\"Train_loss\")\n    ax2.plot(valid_loss, color=\"red\", label=\"Validation_loss\")\n    ax2.set(title=\"loss over epochs\",\n            xlabel=\"Epoch\",\n            ylabel=\"Loss\")\n    ax2.legend()\n\n    fig.show()\n\n    # change the plot style to default\n    plt.style.use('default')\n\n'''\n    function for the training step of the training loop\n'''\ndef train(train_loader,total_outputs, z,model, criterion_ce, criterion_mse, optimizer, device):\n    model.train()\n    running_loss = 0\n\n    for i, (images, labels) in tqdm(enumerate(train_loader),total=len(train_loader)):\n        images = images.to(device)\n        labels = labels.to(device)\n        \n         #meta_images = images + torch.rand(images.size()).to(device) * 1.0 + 0.1 # augment gaussian\n\n        # forward pass\n        outputs = model(images)\n#         with torch.no_grad(): # pytorch tutorial\n#           meta_outputs = model(meta_images) # predict augment image\n        total_outputs[i*(outputs.shape[0]):(i+1)*(outputs.shape[0])]= outputs.data.clone()\n        outputs2 = z[i*(outputs.shape[0]):(i+1)*(outputs.shape[0])].detach().to(device)\n        loss_ce = criterion_ce(outputs, labels)\n        loss_mse = criterion_mse(outputs, outputs2)\n        loss = loss_ce + loss_mse * 0.1 # weight_alpha\n        running_loss += loss.item()\n\n        # backward and optimizer\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    epoch_loss = running_loss / len(train_loader)\n\n    return model, optimizer, epoch_loss,total_outputs\n\n'''\n    function for the validation step of the training loop\n'''\ndef validate(valid_loader, model, criterion, device):\n    model.eval()\n    running_loss = 0\n\n    for images, labels in valid_loader:\n        images = images.to(device)\n        labels = labels.to(device)\n\n        # forward pass and record loss\n        outputs = model(images)\n        \n        loss = criterion(outputs, labels)\n        running_loss += loss.item()\n\n    epoch_loss = running_loss / len(valid_loader)\n\n    return model, epoch_loss\n\n'''\n    function defining the entire training loop\n'''\ndef training_loop(model, criterion_ce, criterion_mse, optimizer, train_loader, valid_loader, epochs, device, print_every=1):\n    if not os.path.exists(\"save_model\"):\n      os.mkdir(\"save_model\")\n    # set object for storing metrics\n    best_loss = 1e10\n    train_losses = []\n    valid_losses = []\n    list_train_acc = []\n    list_val_acc = []\n    alpha=0.6\n    Z = torch.zeros(33500, 10)\n    z = torch.zeros(33500, 10)\n    total_outputs = torch.zeros(33500, 10)\n    # train model\n    for epoch in tqdm(range(0, epochs), total= epochs):\n        # training\n        model, optimizer, train_loss,total_outputs = train(train_loader, total_outputs,z,model, criterion_ce, criterion_mse, optimizer, device)\n        \n        Z = alpha * Z + (1. - alpha) * total_outputs\n        z = Z * (1. / (1. - alpha ** (epoch + 1)))\n\n        # validation\n        with torch.no_grad():\n            model, valid_loss = validate(valid_loader, model, criterion_ce, device)\n\n        if epoch % print_every == print_every - 1:\n            train_acc = get_accuracy(model, train_loader, device=device)\n            valid_acc = get_accuracy(model, valid_loader, device=device)\n\n\n            print('Epochs: {}, Train_loss: {}, Valid_loss: {}, Train_accuracy: {}, Valid_accuracy: {}'.format(\n                    epoch+1, train_loss, valid_loss, train_acc, valid_acc\n                    ))\n\n            list_train_acc.append(train_acc)\n            list_val_acc.append(valid_acc)\n            train_losses.append(train_loss)\n            valid_losses.append(valid_loss)\n            \n#             wandb.log({\"train acc\": train_acc, \"train loss\": train_loss})\n#             wandb.log({\"valid acc\": valid_acc, \"valid loss\": valid_loss})\n            # save model\n            # if train_acc > best_acc: => moi luu\n            torch.save(model.state_dict(), \"save_model/epoch_{}_acc{}.pth\".format(epoch, valid_acc))\n            # best_acc = train_acc\n            # print(\"[INFO] save_model/fold_{}_epoch_{}_acc{}.pth\".format(fold_idx+1, epoch, valid_acc))\n\n    plot_losses(list_train_acc, list_val_acc, train_losses, valid_losses)\n    #wandb.finish()\n\n    return model, optimizer, (train_losses, valid_losses)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:51:11.982081Z","iopub.execute_input":"2023-06-22T03:51:11.982487Z","iopub.status.idle":"2023-06-22T03:51:12.008913Z","shell.execute_reply.started":"2023-06-22T03:51:11.982455Z","shell.execute_reply":"2023-06-22T03:51:12.007690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms(image_size=(224, 224)):\n\n    transforms_train = albumentations.Compose([\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.ImageCompression(quality_lower=99, quality_upper=100),\n        albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n        # albumentations.Resize(image_size, image_size),\n        # albumentations.Cutout(max_h_size=int(image_size * 0.4), max_w_size=int(image_size * 0.4), num_holes=1, p=0.5),\n        albumentations.Normalize()\n    ])\n\n    transforms_val = albumentations.Compose([\n        albumentations.HorizontalFlip(p=0.5),\n        albumentations.ImageCompression(quality_lower=99, quality_upper=100),\n        albumentations.ShiftScaleRotate(shift_limit=0.2, scale_limit=0.2, rotate_limit=10, border_mode=0, p=0.7),\n        # albumentations.Resize(image_size, image_size),\n        # albumentations.Cutout(max_h_size=int(image_size * 0.4), max_w_size=int(image_size * 0.4), num_holes=1, p=0.5),\n        albumentations.Normalize()\n    ])\n\n    return transforms_train, transforms_val","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:53.307257Z","iopub.execute_input":"2023-06-22T03:37:53.307606Z","iopub.status.idle":"2023-06-22T03:37:53.314819Z","shell.execute_reply.started":"2023-06-22T03:37:53.307578Z","shell.execute_reply":"2023-06-22T03:37:53.313598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms_train, transforms_val = get_transforms(image_size=(224, 224))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:37:56.797888Z","iopub.execute_input":"2023-06-22T03:37:56.798254Z","iopub.status.idle":"2023-06-22T03:37:56.803317Z","shell.execute_reply.started":"2023-06-22T03:37:56.798224Z","shell.execute_reply":"2023-06-22T03:37:56.802150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def setup_dataflow(dataset, train_idx, val_idx):\n#     train_sampler = SubsetRandomSampler(train_idx)\n#     val_sampler = SubsetRandomSampler(val_idx)\n\n#     train_loader = DataLoader(dataset, batch_size=32, sampler=train_sampler)\n#     val_loader = DataLoader(dataset, batch_size=32, sampler=val_sampler)\n\n#     return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.710000Z","iopub.status.idle":"2023-06-22T03:31:31.710850Z","shell.execute_reply.started":"2023-06-22T03:31:31.710506Z","shell.execute_reply":"2023-06-22T03:31:31.710529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# class Net(nn.Module):\n#     def __init__(self):\n#         super(Net, self).__init__()\n\n#         self.model = resnet18(num_classes=10)\n#         self.model.conv1 = nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False)\n\n#     def forward(self, x):\n#         return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:38:00.492722Z","iopub.execute_input":"2023-06-22T03:38:00.493080Z","iopub.status.idle":"2023-06-22T03:38:00.499840Z","shell.execute_reply.started":"2023-06-22T03:38:00.493053Z","shell.execute_reply":"2023-06-22T03:38:00.498935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CNN(nn.Module):\n# \tdef __init__(self):\n# \t\tsuper(CNN, self).__init__()\n# \t\tself.conv1 = nn.Conv2d(3, 32, 3)\n# \t\tself.conv2 = nn.Conv2d(32, 64, 3)\n# \t\tself.conv3 = nn.Conv2d(64, 128, 3)\n# \t\tself.conv4 = nn.Conv2d(128, 256, 3)\n# \t\tself.fc1 = nn.Linear(6400, 1024)\n# \t\tself.fc2 = nn.Linear(1024, 512)\n# \t\tself.fc3 = nn.Linear(512, 128)\n# \t\tself.fc4 = nn.Linear(128, 10)\n# \tdef forward(self, x):\n# \t\t# 3 x 32 x 32\n# \t\tx = torch.relu(self.conv1(x))\t\t\t# 32 x 30 x 30\n# \t\tx = nn.BatchNorm2d(32).to(\"cuda\")(x)\t# 32 x 30 x 30\n# \t\tx = torch.relu(self.conv2(x))\t\t\t# 64 x 28 x 28\n# \t\tx = torch.max_pool2d(x, 2)\t\t\t\t# 64 x 14 x 14\n# \t\tx = torch.relu(self.conv3(x))\t\t\t# 128 x 12 x 12\n# \t\tx = nn.BatchNorm2d(128).to(\"cuda\")(x)\t# 128 x 12 x 12\n# \t\tx = torch.relu(self.conv4(x))\t\t\t# 256 x 10 x 10\n# \t\tx = torch.max_pool2d(x, 2)\t\t\t\t# 256 x 5 x 5\n# \t\tx = x.view(-1, 6400)\n# \t\tx = torch.relu(self.fc1(x))\n# \t\tx = torch.tanh(self.fc2(x))\n# \t\tx = torch.relu(self.fc3(x))\n# \t\tx = self.fc4(x)\n# \t\treturn x","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VGG16(nn.Module):\n    def __init__(self, n_classes=10):\n        super(VGG16, self).__init__()\n\n        self.feature_extractor = nn.Sequential(\n            self.create_conv_block([3, 64], [64, 64], [3, 3], [1, 1], 2, 2),\n            self.create_conv_block([64, 128], [128, 128], [3, 3], [1, 1], 2, 2),\n            self.create_conv_block([128, 256, 256], [256, 256, 256], [3, 3, 3], [1, 1, 1], 2, 2),\n            self.create_conv_block([256, 512, 512], [512, 512, 512], [3, 3, 3], [1, 1, 1], 2, 2),\n            self.create_conv_block([512, 512, 512], [512, 512, 512], [3, 3, 3], [1, 1, 1], 2, 2),\n        )\n\n        self.flatten = nn.Flatten()\n\n        self.FC_layer = nn.Sequential(\n            self.create_fc_layer(7*7*512, 1028),\n            self.create_fc_layer(1028, 512),\n        )\n\n        self.classifier = nn.Linear(in_features=512, out_features=n_classes)\n\n    def forward(self, x):\n        output = self.feature_extractor(x)\n        output = self.flatten(output)\n        output = self.FC_layer(output)\n        output = self.classifier(output)\n\n        return output\n\n\n    def create_conv_layer(self, chann_in, chann_out, kernel_size, padding_size):\n        layer = nn.Sequential(\n            nn.Conv2d(chann_in, chann_out, kernel_size=kernel_size, padding=padding_size),\n            nn.BatchNorm2d(chann_out),\n            nn.ReLU()\n        )\n\n        return layer\n\n    def create_conv_block(self, list_chann_in, list_chann_out, list_kernel_size, list_padding_size, pooling_kernel, pooling_Stride):\n        layers = [self.create_conv_layer(list_chann_in[i], list_chann_out[i], list_kernel_size[i], list_padding_size[i])\n                    for i in range(len(list_chann_in))]\n\n        layers += [nn.MaxPool2d(kernel_size=pooling_kernel, stride=pooling_Stride)]\n\n        return nn.Sequential(*layers)\n\n    def create_fc_layer(self, chann_in, chann_out):\n        layer = nn.Sequential(\n            nn.Linear(chann_in, chann_out),\n            nn.ReLU(),\n            # Model with dropout\n            nn.Dropout(0.5),\n        )\n\n        return layer","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# num_folds = 5\n# splits = KFold(n_splits=num_folds,shuffle=True,random_state=42)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:31:31.714850Z","iopub.status.idle":"2023-06-22T03:31:31.715299Z","shell.execute_reply.started":"2023-06-22T03:31:31.715063Z","shell.execute_reply":"2023-06-22T03:31:31.715084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader= DataLoader(train_datas, batch_size= 32, shuffle= True)\nval_loader= DataLoader(val_datas,batch_size=32, shuffle= True)\ntest_loader= DataLoader(test_data,batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:07:56.903350Z","iopub.execute_input":"2023-06-22T05:07:56.903731Z","iopub.status.idle":"2023-06-22T05:07:56.909130Z","shell.execute_reply.started":"2023-06-22T05:07:56.903701Z","shell.execute_reply":"2023-06-22T05:07:56.908071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:38:05.754664Z","iopub.execute_input":"2023-06-22T03:38:05.755126Z","iopub.status.idle":"2023-06-22T03:38:05.762902Z","shell.execute_reply.started":"2023-06-22T03:38:05.755089Z","shell.execute_reply":"2023-06-22T03:38:05.761922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# for fold_idx, (train_idx,val_idx) in enumerate(splits.split(np.arange(len(train_data)))):\n#   print('############### Fold {} ###############'.format(fold_idx + 1))\n#train_loader, val_loader = setup_dataflow(train_data, train_idx, val_idx)\nmodel = VGG16().to(device)\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0001) # 0.0001\nloss_function_ce = nn.CrossEntropyLoss()\nloss_function_mse = nn.MSELoss()\nmodel, optimizer, _ = training_loop(model, loss_function_ce, loss_function_mse, optimizer, train_loader, val_loader, 10, device)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T03:51:17.882807Z","iopub.execute_input":"2023-06-22T03:51:17.883193Z","iopub.status.idle":"2023-06-22T05:04:33.103364Z","shell.execute_reply.started":"2023-06-22T03:51:17.883156Z","shell.execute_reply":"2023-06-22T05:04:33.102336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# testing\nmodel.eval()\nkey = [\"frog\",\"truck\",\"deer\",\"automobile\",\"bird\",\"horse\",\"ship\",\"cat\",\"dog\",\"airplane\"]\nkey = {i : key[i] for i in range(len(key))}\nres = []\n\n\nwith torch.no_grad():\n    for x in test_loader:\n        output = model(x.to(\"cuda\")).to(\"cpu\")\n        output = torch.argmax(output, dim=1)\n        for i in output:\n            res.append(key[i.item()])\ndf = pd.DataFrame(res)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:31:52.707627Z","iopub.execute_input":"2023-06-22T05:31:52.708025Z","iopub.status.idle":"2023-06-22T05:41:42.565500Z","shell.execute_reply.started":"2023-06-22T05:31:52.707993Z","shell.execute_reply":"2023-06-22T05:41:42.564438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:46:13.097512Z","iopub.execute_input":"2023-06-22T05:46:13.098252Z","iopub.status.idle":"2023-06-22T05:46:13.109668Z","shell.execute_reply.started":"2023-06-22T05:46:13.098210Z","shell.execute_reply":"2023-06-22T05:46:13.108739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['id'] = range(1, len(df) + 1)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:46:56.486712Z","iopub.execute_input":"2023-06-22T05:46:56.487389Z","iopub.status.idle":"2023-06-22T05:46:56.493207Z","shell.execute_reply.started":"2023-06-22T05:46:56.487356Z","shell.execute_reply":"2023-06-22T05:46:56.492309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:47:00.585593Z","iopub.execute_input":"2023-06-22T05:47:00.586307Z","iopub.status.idle":"2023-06-22T05:47:00.597468Z","shell.execute_reply.started":"2023-06-22T05:47:00.586275Z","shell.execute_reply":"2023-06-22T05:47:00.596381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.set_index('id')","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:47:27.490957Z","iopub.execute_input":"2023-06-22T05:47:27.491726Z","iopub.status.idle":"2023-06-22T05:47:27.498584Z","shell.execute_reply.started":"2023-06-22T05:47:27.491685Z","shell.execute_reply":"2023-06-22T05:47:27.497700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df\n","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:47:31.132310Z","iopub.execute_input":"2023-06-22T05:47:31.132694Z","iopub.status.idle":"2023-06-22T05:47:31.144512Z","shell.execute_reply.started":"2023-06-22T05:47:31.132656Z","shell.execute_reply":"2023-06-22T05:47:31.143534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=True, header=[\"label\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:47:43.279636Z","iopub.execute_input":"2023-06-22T05:47:43.280031Z","iopub.status.idle":"2023-06-22T05:47:43.991220Z","shell.execute_reply.started":"2023-06-22T05:47:43.280001Z","shell.execute_reply":"2023-06-22T05:47:43.990148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns.values[0] = 'Label'","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:48:48.527599Z","iopub.execute_input":"2023-06-22T05:48:48.528282Z","iopub.status.idle":"2023-06-22T05:48:48.534078Z","shell.execute_reply.started":"2023-06-22T05:48:48.528233Z","shell.execute_reply":"2023-06-22T05:48:48.532934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:48:50.368392Z","iopub.execute_input":"2023-06-22T05:48:50.369086Z","iopub.status.idle":"2023-06-22T05:48:50.379298Z","shell.execute_reply.started":"2023-06-22T05:48:50.369052Z","shell.execute_reply":"2023-06-22T05:48:50.378257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\", index=True, header=[\"label\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-22T05:49:02.679437Z","iopub.execute_input":"2023-06-22T05:49:02.680141Z","iopub.status.idle":"2023-06-22T05:49:03.394943Z","shell.execute_reply.started":"2023-06-22T05:49:02.680106Z","shell.execute_reply":"2023-06-22T05:49:03.393724Z"},"trusted":true},"execution_count":null,"outputs":[]}]}