{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.optim.lr_scheduler as lr_scheduler\nfrom torch.optim.lr_scheduler import _LRScheduler\nimport torch.utils.data as data\n\nimport torchvision.transforms as transforms\nimport torchvision.datasets as datasets\nimport torchvision.models as models\n\nfrom sklearn import decomposition\nfrom sklearn import manifold\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nimport copy\nfrom collections import namedtuple\nimport os\nimport random\nimport shutil\nimport time","metadata":{"id":"NilSkBKhPthJ","execution":{"iopub.status.busy":"2022-02-25T08:20:13.754784Z","iopub.execute_input":"2022-02-25T08:20:13.755391Z","iopub.status.idle":"2022-02-25T08:20:16.354798Z","shell.execute_reply.started":"2022-02-25T08:20:13.755282Z","shell.execute_reply":"2022-02-25T08:20:16.353963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Next, we'll set the random seeds for reproducability.","metadata":{"id":"s7NNtOWlBkXh"}},{"cell_type":"code","source":"import wandb","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.init()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 1234\n\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed(SEED)\ntorch.backends.cudnn.deterministic = True","metadata":{"id":"4QmwmcXuPuLo","execution":{"iopub.status.busy":"2022-02-25T08:20:17.048661Z","iopub.execute_input":"2022-02-25T08:20:17.04893Z","iopub.status.idle":"2022-02-25T08:20:17.056788Z","shell.execute_reply.started":"2022-02-25T08:20:17.048902Z","shell.execute_reply":"2022-02-25T08:20:17.056032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = \"../input/dog-cat-classification/train/train\"\ntest_dir = \"../input/dog-cat-classification/test\"","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:20:20.449177Z","iopub.execute_input":"2022-02-25T08:20:20.449806Z","iopub.status.idle":"2022-02-25T08:20:20.454329Z","shell.execute_reply.started":"2022-02-25T08:20:20.449769Z","shell.execute_reply":"2022-02-25T08:20:20.453328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_size = 600\npretrained_means = [0.485, 0.456, 0.406]\npretrained_stds= [0.229, 0.224, 0.225]\n\n\ntrain_transforms = transforms.Compose([\n                           transforms.Resize(pretrained_size),\n                           transforms.RandomRotation(10),\n                           transforms.GaussianBlur(kernel_size=(5, 5), sigma=(0.1, 1)),\n                           transforms.ColorJitter(brightness=0.5, contrast=0.5, hue=0.5),\n                           transforms.RandomHorizontalFlip(0.5),\n                           transforms.RandomCrop(pretrained_size, padding = 10),\n                           transforms.ToTensor(),\n                           transforms.Normalize(mean = pretrained_means, \n                                                std = pretrained_stds),\n                            \n                       ])\n\ntest_transforms = transforms.Compose([\n                           transforms.Resize(pretrained_size),\n                           transforms.CenterCrop(pretrained_size),\n                           transforms.ToTensor(),\n                           transforms.Normalize(mean = pretrained_means, \n                                                std = pretrained_stds)\n                       ])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = datasets.ImageFolder(root = train_dir, \n                                  transform = train_transforms)\n\ntest_data = datasets.ImageFolder(root = test_dir, \n                                 transform = test_transforms)","metadata":{"id":"l-89cb0XBkXz","execution":{"iopub.status.busy":"2022-02-25T08:20:24.043608Z","iopub.execute_input":"2022-02-25T08:20:24.044138Z","iopub.status.idle":"2022-02-25T08:20:32.8129Z","shell.execute_reply.started":"2022-02-25T08:20:24.044096Z","shell.execute_reply":"2022-02-25T08:20:32.812009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.classes","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:20:32.81632Z","iopub.execute_input":"2022-02-25T08:20:32.817423Z","iopub.status.idle":"2022-02-25T08:20:32.826161Z","shell.execute_reply.started":"2022-02-25T08:20:32.817377Z","shell.execute_reply":"2022-02-25T08:20:32.82537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.classes","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:20:32.827323Z","iopub.execute_input":"2022-02-25T08:20:32.828698Z","iopub.status.idle":"2022-02-25T08:20:32.834281Z","shell.execute_reply.started":"2022-02-25T08:20:32.828666Z","shell.execute_reply":"2022-02-25T08:20:32.833569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VALID_RATIO = 0.95\n\nn_train_examples = int(len(train_data) * VALID_RATIO)\nn_valid_examples = len(train_data) - n_train_examples\n\ntrain_data, valid_data = data.random_split(train_data, \n                                           [n_train_examples, n_valid_examples])","metadata":{"id":"9OQN9jG2P6p8","execution":{"iopub.status.busy":"2022-02-25T08:20:32.836317Z","iopub.execute_input":"2022-02-25T08:20:32.836778Z","iopub.status.idle":"2022-02-25T08:20:32.856235Z","shell.execute_reply.started":"2022-02-25T08:20:32.836742Z","shell.execute_reply":"2022-02-25T08:20:32.85557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"valid_data = copy.deepcopy(valid_data)\nvalid_data.dataset.transform = test_transforms","metadata":{"id":"9A-RD7IdPy7K","execution":{"iopub.status.busy":"2022-02-25T08:20:32.857321Z","iopub.execute_input":"2022-02-25T08:20:32.857988Z","iopub.status.idle":"2022-02-25T08:20:32.989091Z","shell.execute_reply.started":"2022-02-25T08:20:32.857946Z","shell.execute_reply":"2022-02-25T08:20:32.988441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of training examples: {len(train_data)}')\nprint(f'Number of validation examples: {len(valid_data)}')\nprint(f'Number of testing examples: {len(test_data)}')","metadata":{"id":"-4XfHZL1xVLT","outputId":"2a862f4f-25fe-4d58-8604-27ac1722d120","execution":{"iopub.status.busy":"2022-02-25T08:20:32.99039Z","iopub.execute_input":"2022-02-25T08:20:32.990638Z","iopub.status.idle":"2022-02-25T08:20:32.997118Z","shell.execute_reply.started":"2022-02-25T08:20:32.990605Z","shell.execute_reply":"2022-02-25T08:20:32.996331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 16\n\ntrain_iterator = data.DataLoader(train_data, \n                                 shuffle = True, \n                                 batch_size = BATCH_SIZE)\n\nvalid_iterator = data.DataLoader(valid_data, \n                                 batch_size = BATCH_SIZE)\n\ntest_iterator = data.DataLoader(test_data, \n                                batch_size = BATCH_SIZE)","metadata":{"id":"LlF1vQHHRESs","execution":{"iopub.status.busy":"2022-02-25T08:07:20.086434Z","iopub.execute_input":"2022-02-25T08:07:20.08734Z","iopub.status.idle":"2022-02-25T08:07:20.093212Z","shell.execute_reply.started":"2022-02-25T08:07:20.087301Z","shell.execute_reply":"2022-02-25T08:07:20.092316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To ensure the images have been processed correctly we can plot a few of them - ensuring we re-normalize the images so their colors look right.","metadata":{"id":"cwUrAarMBkYA"}},{"cell_type":"code","source":"def normalize_image(image):\n    image_min = image.min()\n    image_max = image.max()\n    image.clamp_(min = image_min, max = image_max)\n    image.add_(-image_min).div_(image_max - image_min + 1e-5)\n    return image    ","metadata":{"id":"MjvuGeRMBkYB","execution":{"iopub.status.busy":"2022-02-25T08:07:20.094612Z","iopub.execute_input":"2022-02-25T08:07:20.095042Z","iopub.status.idle":"2022-02-25T08:07:20.103617Z","shell.execute_reply.started":"2022-02-25T08:07:20.095004Z","shell.execute_reply":"2022-02-25T08:07:20.102945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(images, labels, classes, normalize = True):\n\n    n_images = len(images)\n\n    rows = int(np.sqrt(n_images))\n    cols = int(np.sqrt(n_images))\n\n    fig = plt.figure(figsize = (15, 15))\n\n    for i in range(rows*cols):\n\n        ax = fig.add_subplot(rows, cols, i+1)\n        \n        image = images[i]\n\n        if normalize:\n            image = normalize_image(image)\n\n        ax.imshow(image.permute(1, 2, 0).cpu().numpy())\n        label = classes[labels[i]]\n        ax.set_title(label)\n        ax.axis('off')","metadata":{"id":"XZVxkhKxBkYD","execution":{"iopub.status.busy":"2022-02-25T06:50:17.025123Z","iopub.execute_input":"2022-02-25T06:50:17.025553Z","iopub.status.idle":"2022-02-25T06:50:17.035813Z","shell.execute_reply.started":"2022-02-25T06:50:17.025484Z","shell.execute_reply":"2022-02-25T06:50:17.034546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"N_IMAGES = 25\n\nimages, labels = zip(*[(image, label) for image, label in \n                           [train_data[i] for i in range(N_IMAGES)]])\nclasses = ['cats', 'dogs']\n# classes = test_data.classes\n\n\nplot_images(images, labels, classes)","metadata":{"id":"TueBO2OxRQiq","outputId":"0bbc6ba3-0e1b-47b3-ed1e-1b5dcc373b90","execution":{"iopub.status.busy":"2022-02-25T06:50:17.037581Z","iopub.execute_input":"2022-02-25T06:50:17.038228Z","iopub.status.idle":"2022-02-25T06:50:19.296541Z","shell.execute_reply.started":"2022-02-25T06:50:17.038184Z","shell.execute_reply":"2022-02-25T06:50:19.295309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install efficientnet_pytorch","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:08:49.397602Z","iopub.execute_input":"2022-02-25T08:08:49.397863Z","iopub.status.idle":"2022-02-25T08:09:00.730787Z","shell.execute_reply.started":"2022-02-25T08:08:49.397833Z","shell.execute_reply":"2022-02-25T08:09:00.729965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from efficientnet_pytorch import EfficientNet","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:20:38.237044Z","iopub.execute_input":"2022-02-25T08:20:38.237351Z","iopub.status.idle":"2022-02-25T08:20:38.248813Z","shell.execute_reply.started":"2022-02-25T08:20:38.237318Z","shell.execute_reply":"2022-02-25T08:20:38.247978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EfficientNet.from_pretrained('efficientnet-b7', num_classes=2, dropout_rate = 0.45)","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:20:39.471175Z","iopub.execute_input":"2022-02-25T08:20:39.471814Z","iopub.status.idle":"2022-02-25T08:20:40.756722Z","shell.execute_reply.started":"2022-02-25T08:20:39.471771Z","shell.execute_reply":"2022-02-25T08:20:40.75595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# START_LR = 1e-7\n\n# optimizer = optim.Adam(model.parameters(), lr=START_LR)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ncriterion = nn.CrossEntropyLoss()\n\nmodel = model.to(device)\ncriterion = criterion.to(device)","metadata":{"id":"4Cuqluk1ZNOW","execution":{"iopub.status.busy":"2022-02-25T08:20:42.02255Z","iopub.execute_input":"2022-02-25T08:20:42.023211Z","iopub.status.idle":"2022-02-25T08:20:45.05672Z","shell.execute_reply.started":"2022-02-25T08:20:42.02317Z","shell.execute_reply":"2022-02-25T08:20:45.055898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"FOUND_LR = 0.2*1e-4\noptimizer = optim.AdamW(model.parameters(), lr = FOUND_LR)","metadata":{"id":"fDjS7JvNBkZR","execution":{"iopub.status.busy":"2022-02-25T06:51:11.213878Z","iopub.execute_input":"2022-02-25T06:51:11.214197Z","iopub.status.idle":"2022-02-25T06:51:11.225522Z","shell.execute_reply.started":"2022-02-25T06:51:11.214165Z","shell.execute_reply":"2022-02-25T06:51:11.22438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 20\nSTEPS_PER_EPOCH = len(train_iterator)\nTOTAL_STEPS = EPOCHS * STEPS_PER_EPOCH\n\nMAX_LRS = [p['lr'] for p in optimizer.param_groups]\n\nscheduler = lr_scheduler.OneCycleLR(optimizer,\n                                    max_lr = MAX_LRS,\n                                    total_steps = TOTAL_STEPS)","metadata":{"id":"riS8zV2IxVMK","execution":{"iopub.status.busy":"2022-02-25T06:51:21.047229Z","iopub.execute_input":"2022-02-25T06:51:21.04792Z","iopub.status.idle":"2022-02-25T06:51:21.053767Z","shell.execute_reply.started":"2022-02-25T06:51:21.047883Z","shell.execute_reply":"2022-02-25T06:51:21.052644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import precision_score, \\\n    recall_score, confusion_matrix, classification_report, \\\n    accuracy_score, f1_score","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:09:56.784122Z","iopub.execute_input":"2022-02-25T08:09:56.784556Z","iopub.status.idle":"2022-02-25T08:09:56.788852Z","shell.execute_reply.started":"2022-02-25T08:09:56.784517Z","shell.execute_reply":"2022-02-25T08:09:56.788098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-02-25T08:09:56.791123Z","iopub.execute_input":"2022-02-25T08:09:56.791884Z","iopub.status.idle":"2022-02-25T08:09:56.798936Z","shell.execute_reply.started":"2022-02-25T08:09:56.791846Z","shell.execute_reply":"2022-02-25T08:09:56.798295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, iterator, optimizer, criterion, scheduler, device):\n    \n    epoch_loss = 0\n    epoch_acc_1 = 0\n    epoch_acc_5 = 0\n    \n    model.train()\n    \n    for (x, y) in tqdm(iterator):\n        \n        x = x.to(device)\n        y = y.to(device)\n        \n        optimizer.zero_grad()\n                \n        y_pred = model(x)\n        \n        loss = criterion(y_pred, y)\n        \n        loss.backward()\n        \n        optimizer.step()\n        \n        scheduler.step()\n        \n        epoch_loss += loss.item()\n        \n    epoch_loss /= len(iterator)\n    wandb.log({\"Train Loss\": epoch_loss})\n        \n        \n    return epoch_loss","metadata":{"id":"Qr6Je0jMafsp","execution":{"iopub.status.busy":"2022-02-25T06:51:29.412337Z","iopub.execute_input":"2022-02-25T06:51:29.413139Z","iopub.status.idle":"2022-02-25T06:51:29.421676Z","shell.execute_reply.started":"2022-02-25T06:51:29.413096Z","shell.execute_reply":"2022-02-25T06:51:29.419731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate(model, iterator, criterion, device):\n    \n    epoch_loss = 0\n    epoch_acc_1 = 0\n    epoch_acc_5 = 0\n    \n    model.eval()\n    total_preds = []\n    total_groundtruth = []\n    \n    with torch.no_grad():\n        \n        for (x, y) in tqdm(iterator):\n\n            x = x.to(device)\n            y = y.to(device)\n\n            y_pred = model(x)\n\n            loss = criterion(y_pred, y)\n\n            epoch_loss += loss.item()\n            \n            np_y_pred = y_pred.detach().cpu().numpy()\n\n            np_y = y.detach().cpu().numpy()\n\n            total_preds.append(np_y_pred)\n            total_groundtruth.append(np_y)\n        \n    epoch_loss /= len(iterator)\n    \n    total_preds  = np.concatenate(total_preds, axis=0,)\n    total_preds = np.argmax(total_preds, axis=1)\n    total_preds = np.array(total_preds, dtype = np.int16)\n    total_groundtruth = np.concatenate(total_groundtruth, axis = 0)\n    total_groundtruth = np.array(total_groundtruth, dtype = np.int16)\n    focus_f1 = f1_score(total_groundtruth, total_preds)\n    acc = accuracy_score(total_groundtruth, total_preds)\n    recall = recall_score(total_groundtruth, total_preds)\n    precision = precision_score(total_groundtruth, total_preds)\n    print(\"Accuracy: \", acc)\n    print(\"F1 score: \", focus_f1)\n    print('Recall:', recall)\n    print('Precision:', precision)\n    print('\\n clasification report:\\n', classification_report(total_groundtruth,total_preds))\n    print('\\n confussion matrix:\\n',confusion_matrix(total_groundtruth, total_preds))\n    wandb.log({\"F1\": focus_f1})\n    wandb.log({\"Precision\": precision})\n    wandb.log({\"Recall\": recall})\n    wandb.log({\"Val Loss\": epoch_loss})\n        \n    return epoch_loss , focus_f1","metadata":{"id":"85FW74KaakI_","execution":{"iopub.status.busy":"2022-02-25T06:51:31.939927Z","iopub.execute_input":"2022-02-25T06:51:31.940241Z","iopub.status.idle":"2022-02-25T06:51:31.953131Z","shell.execute_reply.started":"2022-02-25T06:51:31.940209Z","shell.execute_reply":"2022-02-25T06:51:31.951718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def epoch_time(start_time, end_time):\n    elapsed_time = end_time - start_time\n    elapsed_mins = int(elapsed_time / 60)\n    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n    return elapsed_mins, elapsed_secs","metadata":{"id":"wkR6LzakamOV","execution":{"iopub.status.busy":"2022-02-25T06:51:35.170294Z","iopub.execute_input":"2022-02-25T06:51:35.171094Z","iopub.status.idle":"2022-02-25T06:51:35.176872Z","shell.execute_reply.started":"2022-02-25T06:51:35.171059Z","shell.execute_reply":"2022-02-25T06:51:35.175602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_valid_loss = float('inf')\n\nfor epoch in range(EPOCHS):\n    \n    start_time = time.monotonic()\n    \n    train_loss = train(model, train_iterator, optimizer, criterion, scheduler, device)\n    valid_loss, focus_f1 = evaluate(model, valid_iterator, criterion, device)\n        \n    if valid_loss < best_valid_loss:\n        best_valid_loss = valid_loss\n        torch.save(model.state_dict(), 'tut5-model.pt')\n\n    end_time = time.monotonic()\n\n    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n    \n    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')","metadata":{"id":"lvoyX823apEN","outputId":"0cb310cb-9016-4cca-dcbb-df2f04345e99","execution":{"iopub.status.busy":"2022-02-25T06:51:37.141387Z","iopub.execute_input":"2022-02-25T06:51:37.142221Z","iopub.status.idle":"2022-02-25T07:10:47.780185Z","shell.execute_reply.started":"2022-02-25T06:51:37.142186Z","shell.execute_reply":"2022-02-25T07:10:47.778037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = EfficientNet.from_pretrained('efficientnet-b7', num_classes=2)\nmodel.load_state_dict(torch.load(\"/kaggle/working/tut5-model.pt\"))\nmodel.to(device)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(model, iterator):\n\n    model.eval()\n    preds = []\n\n    with torch.no_grad():\n        for (x, y) in tqdm(iterator):\n            x = x.to(device)\n\n            y_pred = model(x)\n\n            y_prob = F.softmax(y_pred, dim = -1)\n            top_pred = y_prob.argmax(1, keepdim = True)\n\n            preds.append(top_pred.detach().cpu().numpy())\n            \n    preds = np.concatenate(preds, axis = 0)\n    print(np.shape(preds))\n    return preds","metadata":{"id":"AlCBiYxQbi4s","execution":{"iopub.status.busy":"2022-02-25T08:21:16.318414Z","iopub.execute_input":"2022-02-25T08:21:16.319147Z","iopub.status.idle":"2022-02-25T08:21:16.32736Z","shell.execute_reply.started":"2022-02-25T08:21:16.319106Z","shell.execute_reply":"2022-02-25T08:21:16.326478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = get_predictions(model, test_iterator)","metadata":{"id":"JmfK-IZ6bmlz","execution":{"iopub.status.busy":"2022-02-25T08:21:18.469464Z","iopub.execute_input":"2022-02-25T08:21:18.470122Z","iopub.status.idle":"2022-02-25T08:21:18.603021Z","shell.execute_reply.started":"2022-02-25T08:21:18.470084Z","shell.execute_reply":"2022-02-25T08:21:18.602109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\npredict_dataframe = pd.read_csv(\"/kaggle/input/dog-cat-classification/sample_submission.csv\")\nl_predict = [pred[0] for pred in preds]\npredict_dataframe[\"labels\"] = l_predict\npredict_dataframe.to_csv(\"/kaggle/working/Submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}