{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### **Setup**","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"%%bash\npip install timm -q","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:46:31.397841Z","iopub.execute_input":"2022-09-24T20:46:31.39877Z","iopub.status.idle":"2022-09-24T20:46:44.597669Z","shell.execute_reply.started":"2022-09-24T20:46:31.398662Z","shell.execute_reply":"2022-09-24T20:46:44.596116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Library Imports**","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport cv2\nimport timm\nimport torch\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nfrom time import time\nfrom scipy import stats\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader as DL\nfrom torchvision import models, transforms\n\n# from sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:46:44.600329Z","iopub.execute_input":"2022-09-24T20:46:44.600756Z","iopub.status.idle":"2022-09-24T20:46:48.163926Z","shell.execute_reply.started":"2022-09-24T20:46:44.600714Z","shell.execute_reply":"2022-09-24T20:46:48.162563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Utilities and Helpers**","metadata":{}},{"cell_type":"code","source":"SEED: int = 42\nSIZE: int = 384\nMODEL_PATHS: str = \"../input/4ac-en4-a384\"\n\n\ndef breaker(num: int=50, char: str=\"*\") -> None:\n    print(\"\\n\" + num*char + \"\\n\")\n\n    \ndef get_image(path: str, size: int=224) -> np.ndarray:\n    image = cv2.imread(path, cv2.IMREAD_COLOR)\n    image = cv2.cvtColor(src=image, code=cv2.COLOR_BGR2RGB)\n    return cv2.resize(src=image, dsize=(size, size), interpolation=cv2.INTER_AREA)\n\n\n# def show_loss_graphs(L: list) -> None:\n#     TL, VL = [], []\n#     for i in range(len(L)):\n#         TL.append(L[i][\"train\"])\n#         VL.append(L[i][\"valid\"])\n#     x_Axis = np.arange(1, len(TL) + 1)\n#     plt.figure()\n#     plt.plot(x_Axis, TL, \"r\", label=\"Train\")\n#     plt.plot(x_Axis, VL, \"b\", label=\"Valid\")\n#     plt.legend()\n#     plt.grid()\n#     plt.title(\"Loss Graph\")\n#     plt.show()\n\n    \n# def show_accuracy_graphs(A: list) -> None:\n#     TA, VA = [], []\n#     for i in range(len(A)):\n#         TA.append(A[i][\"train\"])\n#         VA.append(A[i][\"valid\"])\n#     x_Axis = np.arange(1, len(TA) + 1)\n#     plt.figure()\n#     plt.plot(x_Axis, TA, \"r\", label=\"Train\")\n#     plt.plot(x_Axis, VA, \"b\", label=\"Valid\")\n#     plt.legend()\n#     plt.grid()\n#     plt.title(\"Accuracy Graph\")\n#     plt.show()\n    \n\n# def show_lr_graph(LR: list) -> None:\n#     x_Axis = [i+1 for i in range(len(LR))]\n#     plt.figure(figsize=(8, 6))\n#     plt.plot(x_Axis, LR, \"rx\")\n#     plt.grid()\n#     plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:09.700014Z","iopub.execute_input":"2022-09-24T20:47:09.70038Z","iopub.status.idle":"2022-09-24T20:47:09.710606Z","shell.execute_reply.started":"2022-09-24T20:47:09.700351Z","shell.execute_reply":"2022-09-24T20:47:09.709664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Configuration**","metadata":{}},{"cell_type":"code","source":"class CFG(object):\n    def __init__(self, \n                 seed: int = 42,\n                 size: int = 224,\n                 n_splits: int = 5,\n                 batch_size: int = 16,\n                 epochs: int = 25,\n                 early_stopping: int = 5,\n                 lr: float = 1e-4,\n                 wd: float = 0.0,\n                 max_lr: float = 1e-3,\n                 pct_start: float = 0.2,\n                 steps_per_epoch: int = 100,\n                 div_factor: int = 1e3, \n                 final_div_factor: float = 1e3,\n                 ):\n        self.seed = seed\n        self.size = size\n        self.n_splits = n_splits\n        self.batch_size = batch_size\n        self.epochs = epochs\n        self.early_stopping = early_stopping\n        self.lr = lr\n        self.wd = wd\n        self.max_lr = max_lr\n        self.pct_start = pct_start\n        self.steps_per_epoch = steps_per_epoch\n        self.div_factor = div_factor\n        self.final_div_factor = final_div_factor\n        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        self.train_transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize([0.48805, 0.46218, 0.38043], [0.21471, 0.21141, 0.21124]),\n            transforms.RandomAffine(degrees=(-45, 45), translate=(0.15, 0.15), scale=(0.5, 1.5)),\n            transforms.RandomHorizontalFlip(p=0.25),\n            transforms.RandomVerticalFlip(p=0.25),\n        ])\n        self.valid_transform = transforms.Compose([\n            transforms.ToTensor(),\n            transforms.Normalize([0.48805, 0.46218, 0.38043], [0.21471, 0.21141, 0.21124]),\n        ])\n                                \n        self.save_path = \"saves\"\n        if not os.path.exists(self.save_path): os.makedirs(self.save_path)\n    \ncfg = CFG(seed=SEED, size=SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:11.292552Z","iopub.execute_input":"2022-09-24T20:47:11.293428Z","iopub.status.idle":"2022-09-24T20:47:11.370739Z","shell.execute_reply.started":"2022-09-24T20:47:11.293392Z","shell.execute_reply":"2022-09-24T20:47:11.369614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Dataset Template**","metadata":{}},{"cell_type":"code","source":"# class DS(Dataset):\n#     def __init__(\n#         self, \n#         base_path: str,\n#         filenames: np.ndarray, \n#         folder_names: np.ndarray, \n#         labels: np.ndarray, \n#         size: int,\n#         transform=None):\n        \n#         self.base_path = base_path\n#         self.filenames = filenames\n#         self.folder_names = folder_names\n#         self.labels = labels\n#         self.size = size\n#         self.transform = transform\n    \n#     def __len__(self):\n#         return self.filenames.shape[0]\n    \n#     def __getitem__(self, idx):\n#         path = os.path.join(os.path.join(self.base_path, self.folder_names[idx]), self.filenames[idx])\n#         image = get_image(path, self.size)\n#         return self.transform(image), self.labels[idx]","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:13.079011Z","iopub.execute_input":"2022-09-24T20:47:13.079419Z","iopub.status.idle":"2022-09-24T20:47:13.084317Z","shell.execute_reply.started":"2022-09-24T20:47:13.079389Z","shell.execute_reply":"2022-09-24T20:47:13.083314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DS(Dataset):\n    def __init__(\n        self, \n        base_path: str,\n        filenames: np.ndarray, \n        size: int,\n        transform=None):\n        \n        self.base_path = base_path\n        self.filenames = filenames\n        self.size = size\n        self.transform = transform\n    \n    def __len__(self):\n        return self.filenames.shape[0]\n    \n    def __getitem__(self, idx):\n        path = os.path.join(self.base_path, str(self.filenames[idx]) + \".jpg\")\n        image = get_image(path, self.size)\n        return self.transform(image)","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:13.210521Z","iopub.execute_input":"2022-09-24T20:47:13.211084Z","iopub.status.idle":"2022-09-24T20:47:13.217527Z","shell.execute_reply.started":"2022-09-24T20:47:13.211054Z","shell.execute_reply":"2022-09-24T20:47:13.21653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Model**","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super(Model, self).__init__()\n            \n        self.model = timm.create_model(model_name=\"efficientnet_b4\", pretrained=False)\n        self.model.classifier = nn.Linear(in_features=self.model.classifier.in_features, out_features=4)\n            \n    def freeze(self):\n        for params in self.parameters(): params.requires_grad = False\n\n    def forward(self, x):\n        return nn.LogSoftmax(dim=1)(self.model(x))","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:17.881948Z","iopub.execute_input":"2022-09-24T20:47:17.882304Z","iopub.status.idle":"2022-09-24T20:47:17.893194Z","shell.execute_reply.started":"2022-09-24T20:47:17.882275Z","shell.execute_reply":"2022-09-24T20:47:17.891245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Fit and Predict**","metadata":{}},{"cell_type":"code","source":"# def fit(model=None,\n#         optimizer=None, \n#         scheduler_rlrop=None,\n#         scheduler_oclr=None,\n#         epochs=None, \n#         early_stopping_patience=None, \n#         dataloaders=None, \n#         fold=None, \n#         save_path=None,\n#         device=None,\n#         verbose=False) -> tuple:\n    \n#     def get_accuracy(y_pred, y_true):\n#         y_pred = torch.argmax(y_pred, dim=1)\n#         return torch.count_nonzero(y_pred == y_true).item() / len(y_pred)\n    \n    \n#     if verbose:\n#         breaker()\n#         if fold: print(f\"Training Fold {fold}...\")\n#         else: print(\"Training ...\")\n#         breaker()\n        \n#     bestLoss: dict = {\"train\" : np.inf, \"valid\" : np.inf} \n#     bestAccs: dict = {\"train\" : 0.0, \"valid\" : 0.0}\n        \n#     Losses: list = []\n#     Accuracies: list = [] \n#     LRs: list = []\n        \n#     if fold: \n#         ble_name = f\"ble_state_fold_{fold}.pt\"\n#         bae_name = f\"bae_state_fold_{fold}.pt\"\n#     else: \n#         ble_name = f\"ble_state.pt\"\n#         bae_name = f\"bae_state.pt\"\n    \n#     start_time = time()\n#     for e in range(epochs):\n#         e_st = time()\n#         epochLoss: dict = {\"train\" : 0.0, \"valid\" : 0.0} \n#         epochAccs: dict = {\"train\" : 0.0, \"valid\" : 0.0}\n\n#         for phase in [\"train\", \"valid\"]:\n#             if phase == \"train\":\n#                 model.train()\n#             else:\n#                 model.eval()\n            \n#             lossPerPass: list = []\n#             accsPerPass: list = []\n                \n#             for X, y in dataloaders[phase]:\n#                 X, y = X.to(device), y.to(device).view(-1)\n\n#                 optimizer.zero_grad()\n#                 with torch.set_grad_enabled(phase == \"train\"):\n#                     output = model(X)\n#                     loss = torch.nn.NLLLoss()(output, y)\n#                     if phase == \"train\":\n#                         loss.backward()\n#                         optimizer.step()\n#                         if scheduler_oclr: scheduler_oclr.step()\n#                 lossPerPass.append(loss.item())\n#                 accsPerPass.append(get_accuracy(output, y))\n#             epochLoss[phase] = np.mean(np.array(lossPerPass))\n#             epochAccs[phase] = np.mean(np.array(accsPerPass))\n#         if scheduler_oclr: LRs.append(scheduler_oclr.get_last_lr())\n#         Losses.append(epochLoss)\n#         Accuracies.append(epochAccs)\n    \n#         if scheduler_oclr:\n#             save_dict = {\"model_state_dict\"     : model.state_dict(),\n#                          \"optim_state_dict\"     : optimizer.state_dict(),\n#                          \"scheduler_state_dict\" : scheduler_oclr.state_dict()}\n        \n#         elif scheduler_rlrop:\n#             save_dict = {\"model_state_dict\"     : model.state_dict(),\n#                          \"optim_state_dict\"     : optimizer.state_dict(),\n#                          \"scheduler_state_dict\" : scheduler_rlrop.state_dict()}\n        \n#         else:\n#             save_dict = {\"model_state_dict\"     : model.state_dict(),\n#                          \"optim_state_dict\"     : optimizer.state_dict()}\n        \n        \n#         if early_stopping_patience:\n#             if epochLoss[\"valid\"] < bestLoss[\"valid\"]:\n#                 bestLoss = epochLoss\n#                 BLE = e + 1\n#                 torch.save(save_dict, os.path.join(save_path, ble_name))\n#                 torch.save(save_dict, os.path.join(save_path, bae_name))\n#                 early_stopping_step = 0\n#             else:\n#                 early_stopping_step += 1\n#                 if early_stopping_step > early_stopping_patience:\n#                     print(\"\\nEarly Stopping at Epoch {}\".format(e + 1))\n#                     break\n        \n#         if epochLoss[\"valid\"] < bestLoss[\"valid\"]:\n#             bestLoss = epochLoss\n#             BLE = e + 1\n#             torch.save(save_dict,os.path.join(save_path, ble_name))\n        \n#         if epochAccs[\"valid\"] > bestAccs[\"valid\"]:\n#             bestAccs = epochAccs\n#             BAE = e + 1\n#             torch.save(save_dict,os.path.join(save_path, bae_name))\n        \n#         if scheduler_rlrop: scheduler_rlrop.step(epochLoss[\"valid\"])\n        \n#         if verbose:\n#             print(\"Epoch: {} | Train Loss: {:.5f} | Valid Loss: {:.5f} |\\\n#  Train Accs: {:.5f} | Valid Accs: {:.5f} | Time: {:.2f} seconds\".format(e+1, \n#                                                                         epochLoss[\"train\"], epochLoss[\"valid\"], \n#                                                                         epochAccs[\"train\"], epochAccs[\"valid\"], \n#                                                                         time()-e_st))\n\n#     if verbose:                                           \n#         breaker()\n#         print(f\"Best Validation Loss at Epoch {BLE}\")\n#         breaker()\n#         print(f\"Best Validation Accs at Epoch {BAE}\")\n#         breaker()\n#         print(\"Time Taken [{} Epochs] : {:.2f} minutes\".format(len(Losses), (time()-start_time)/60))\n    \n#     return Losses, Accuracies, LRs, BLE, BAE, ble_name, bae_name, bestLoss, bestAccs\n\n\ndef predict_batch(model=None, dataloader=None, path=None, device=None) -> np.ndarray:\n    model.load_state_dict(torch.load(path, map_location=device)[\"model_state_dict\"])\n    model.to(device)    \n    model.eval()\n    \n    y_pred = torch.zeros(1, 1).to(device)\n    \n    for X in dataloader:\n        X = X.to(device)\n        with torch.no_grad():\n            output = torch.argmax(torch.exp(model(X)), dim=1)\n        y_pred = torch.cat((y_pred, output.view(-1, 1)), dim=0)\n    \n    return y_pred[1:].detach().cpu().numpy()","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:18.497489Z","iopub.execute_input":"2022-09-24T20:47:18.497924Z","iopub.status.idle":"2022-09-24T20:47:18.51186Z","shell.execute_reply.started":"2022-09-24T20:47:18.497892Z","shell.execute_reply":"2022-09-24T20:47:18.510573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Train**","metadata":{}},{"cell_type":"code","source":"# df = pd.read_csv(\"../input/4ac-dataframe/data.csv\")\n\n# filenames = df.filenames.copy().values\n# folder_names = df.folder_names.copy().values\n# labels = df.classes.copy().values","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:19.229512Z","iopub.execute_input":"2022-09-24T20:47:19.230484Z","iopub.status.idle":"2022-09-24T20:47:19.235676Z","shell.execute_reply.started":"2022-09-24T20:47:19.230441Z","shell.execute_reply":"2022-09-24T20:47:19.234517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fold: int = 1\n# BLs: list = []\n# BAs: list = []\n    \n# for tr_idx, va_idx in StratifiedKFold(n_splits=cfg.n_splits, random_state=cfg.seed, shuffle=True).split(filenames, labels):\n\n#     tr_filenames, va_filenames       = filenames[tr_idx], filenames[va_idx] \n#     tr_folder_names, va_folder_names = folder_names[tr_idx], folder_names[va_idx] \n#     tr_labels, va_labels             = labels[tr_idx], labels[va_idx]\n    \n#     tr_data_setup = DS(\n#         base_path=\"../input/4-animal-classification/train\", \n#         filenames=tr_filenames, \n#         folder_names=tr_folder_names,\n#         labels=tr_labels.reshape(-1, 1),\n#         size=cfg.size,\n#         transform=cfg.train_transform)\n    \n#     va_data_setup = DS(\n#         base_path=\"../input/4-animal-classification/train\", \n#         filenames=va_filenames, \n#         folder_names=va_folder_names,\n#         labels=va_labels.reshape(-1, 1),\n#         size=cfg.size,\n#         transform=cfg.valid_transform)\n\n#     dataloaders = {\n#         \"train\" : DL(tr_data_setup, batch_size=cfg.batch_size, shuffle=True, generator=torch.manual_seed(cfg.seed)),\n#         \"valid\" : DL(va_data_setup, batch_size=cfg.batch_size, shuffle=False),\n#     }\n\n#     cfg = CFG(\n#         epochs=25, \n#         batch_size=64, \n#         early_stopping=5,\n#         steps_per_epoch=len(dataloaders[\"train\"])\n#     )\n    \n#     torch.manual_seed(cfg.seed)\n#     model = Model().to(cfg.device)\n\n#     optimizer = optim.Adam([p for p in model.parameters() if p.requires_grad], lr=cfg.lr, weight_decay=cfg.wd)\n#     # optimizer = optim.AdamW([p for p in model.parameters() if p.requires_grad], lr=cfg.lr, weight_decay=cfg.wd)\n#     # optimizer = optim.SGD([p for p in model.parameters() if p.requires_grad], lr=cfg.lr, weight_decay=cfg.wd, momentum=0.9)\n    \n#     scheduler_oclr = optim.lr_scheduler.OneCycleLR(\n#         optimizer=optimizer, \n#         max_lr=cfg.max_lr, \n#         epochs=cfg.epochs, \n#         steps_per_epoch=cfg.steps_per_epoch,\n#         pct_start=cfg.pct_start, \n#         div_factor=cfg.div_factor, \n#         final_div_factor=cfg.final_div_factor\n#     )\n\n#     # scheduler_rlrop = optim.lr_scheduler.ReduceLROnPlateau(\n#     #     optimizer=optimizer,\n#     #     patience=cfg.patience,\n#     #     eps=cfg.eps,\n#     #     verbose=True\n#     # )\n\n#     # scheduler_oclr = None\n#     scheduler_rlrop = None\n    \n#     L, A, LR, BLE, BAE, _, _, BL, BA = fit(\n#         model=model, \n#         optimizer=optimizer, \n#         scheduler_oclr=scheduler_oclr,\n#         scheduler_rlrop=scheduler_rlrop,\n#         epochs=cfg.epochs, \n#         early_stopping_patience=cfg.early_stopping, \n#         dataloaders=dataloaders, \n#         device=cfg.device,\n#         save_path=cfg.save_path,\n#         fold=fold,\n#         verbose=True\n#     )\n    \n#     breaker()\n#     show_loss_graphs(L)\n#     breaker()\n#     show_accuracy_graphs(A)\n#     breaker()\n#     if scheduler_oclr:\n#         show_lr_graph(LR)\n#         breaker()\n    \n#     BLs.append(BL)\n#     BAs.append(BA)\n    \n#     fold += 1","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:47:19.490469Z","iopub.execute_input":"2022-09-24T20:47:19.49082Z","iopub.status.idle":"2022-09-24T20:47:19.49728Z","shell.execute_reply.started":"2022-09-24T20:47:19.490791Z","shell.execute_reply":"2022-09-24T20:47:19.496142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Submission**","metadata":{}},{"cell_type":"code","source":"torch.manual_seed(cfg.seed)\nmodel = Model().to(cfg.device)\n\nss_df = pd.read_csv(\"../input/4-animal-classification/Sample_submission.csv\")\nts_filenames = ss_df[\"ID\"]\n\nts_data_setup = DS(\n    base_path=\"../input/4-animal-classification/test/test\", \n    filenames=ts_filenames, \n    size=cfg.size,\n    transform=cfg.train_transform\n)\nts_data = DL(ts_data_setup, batch_size=cfg.batch_size, shuffle=False)\n\ny_pred = np.zeros((1, len(ts_filenames)), dtype=np.uint8)\n\npaths = os.listdir(MODEL_PATHS + \"/saves\")\nfor path in paths:\n    temp = predict_batch(\n        model=model,\n        dataloader=ts_data,\n        path=os.path.join(MODEL_PATHS + \"/saves\", path),\n        device=cfg.device\n    )\n    y_pred = np.concatenate((y_pred, temp.reshape(1, -1)), axis=0)\n\n\ny_pred = y_pred[1:]\ny_pred = stats.mode(y_pred)[0]\n\nss_df[\"Label\"] = y_pred.reshape(-1).astype(\"uint8\")\nss_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-24T20:51:23.03716Z","iopub.execute_input":"2022-09-24T20:51:23.037607Z","iopub.status.idle":"2022-09-24T20:51:45.242897Z","shell.execute_reply.started":"2022-09-24T20:51:23.037574Z","shell.execute_reply":"2022-09-24T20:51:45.241794Z"},"trusted":true},"execution_count":null,"outputs":[]}]}