{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torchsummary -q","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-30T01:46:43.853596Z","iopub.execute_input":"2023-03-30T01:46:43.854581Z","iopub.status.idle":"2023-03-30T01:46:58.126918Z","shell.execute_reply.started":"2023-03-30T01:46:43.854541Z","shell.execute_reply":"2023-03-30T01:46:58.125290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport matplotlib.pyplot as plt\n\nimport time\nimport random\nimport os\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam, AdamW, RMSprop\nfrom torch.optim.lr_scheduler import CosineAnnealingLR, ReduceLROnPlateau\nfrom torchsummary import summary\n\nimport albumentations as A","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:46:58.129640Z","iopub.execute_input":"2023-03-30T01:46:58.130142Z","iopub.status.idle":"2023-03-30T01:47:03.529946Z","shell.execute_reply.started":"2023-03-30T01:46:58.130099Z","shell.execute_reply":"2023-03-30T01:47:03.527851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_PATH = '/kaggle/input/digit-recognizer/'\nOUTPUT_PATH = './'\n\ntrain = pl.read_csv(INPUT_PATH + 'train.csv')\ntest = pl.read_csv(INPUT_PATH + 'test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:47:03.532249Z","iopub.execute_input":"2023-03-30T01:47:03.533684Z","iopub.status.idle":"2023-03-30T01:47:05.622022Z","shell.execute_reply.started":"2023-03-30T01:47:03.533598Z","shell.execute_reply":"2023-03-30T01:47:05.620635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train shape:\", train.shape)\nprint(\"Test shape:\", test.shape)\n\n\nprint(\"\\nTrain dataset:\")\nprint(train.head())\nprint(\"\\nTest dataset:\")\nprint(test.head())","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:47:05.626818Z","iopub.execute_input":"2023-03-30T01:47:05.627333Z","iopub.status.idle":"2023-03-30T01:47:05.655448Z","shell.execute_reply.started":"2023-03-30T01:47:05.627289Z","shell.execute_reply":"2023-03-30T01:47:05.654313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_nulls = 0\nfor col in train.get_columns():\n    train_nulls += col.is_null().sum()\nprint(\"Missing values in train dataset:\", train_nulls)\n\ntest_nulls = 0\nfor col in train.get_columns():\n    test_nulls += col.is_null().sum()\nprint(\"Missing values in test dataset:\", test_nulls)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:47:05.656736Z","iopub.execute_input":"2023-03-30T01:47:05.658193Z","iopub.status.idle":"2023-03-30T01:47:05.974160Z","shell.execute_reply.started":"2023-03-30T01:47:05.658148Z","shell.execute_reply":"2023-03-30T01:47:05.972748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Config:\n    DEBUG = False\n    \n    IMG_HEIGHT = 28\n    IMG_WIDTH = 28\n    N_CLASS = len(np.unique(train['label']))\n    \n    split_fraction = 0.95\n    \n    print_freq = 100\n    BATCH_SIZE = 1024\n    EPOCHS = 30\n    \n    save_best_loss = False\n    save_best_accuracy = True\n    \n    optimizer = 'rmsprop'\n    learning_rate = 1e-3\n    weight_decay = 0.1\n    l2_penalty = 0.01\n    rms_momentum = 0\n    \n    scheduler = 'ReduceLROnPlateau'\n    plateau_factor = 0.5\n    plateau_patience = 3\n    cosine_T_max = 4\n    cosine_eta_min = 1e-8\n    verbose = True\n    \n    shuffle = False\n    \n    probability = 0.6\n    \n    random_seed = 42","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:47:05.975690Z","iopub.execute_input":"2023-03-30T01:47:05.976083Z","iopub.status.idle":"2023-03-30T01:47:05.997856Z","shell.execute_reply.started":"2023-03-30T01:47:05.976045Z","shell.execute_reply":"2023-03-30T01:47:05.996237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if Config.DEBUG:\n    Config.EPOCHS = 10\n    train = train.sampler(frac = 0.5).reset_index(drop=True)\n    Config.N_CLASS = len(np.unique(train['label']))","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:47:06.002073Z","iopub.execute_input":"2023-03-30T01:47:06.003068Z","iopub.status.idle":"2023-03-30T01:47:06.009300Z","shell.execute_reply.started":"2023-03-30T01:47:06.003026Z","shell.execute_reply":"2023-03-30T01:47:06.008005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"DEBUG:\", Config.DEBUG)\nprint(\"Train Shape:\", train.shape)\nprint(\"Test Shape:\", test.shape)\n\nprint(\"\\nImage shape (H, W):\", Config.IMG_HEIGHT, Config.IMG_WIDTH)\nprint(\"Number of classes in train dataset:\", Config.N_CLASS)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:48:13.441051Z","iopub.execute_input":"2023-03-30T01:48:13.441957Z","iopub.status.idle":"2023-03-30T01:48:13.456262Z","shell.execute_reply.started":"2023-03-30T01:48:13.441912Z","shell.execute_reply":"2023-03-30T01:48:13.454201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:48:55.639828Z","iopub.execute_input":"2023-03-30T01:48:55.640321Z","iopub.status.idle":"2023-03-30T01:48:55.648814Z","shell.execute_reply.started":"2023-03-30T01:48:55.640282Z","shell.execute_reply":"2023-03-30T01:48:55.646807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_torch(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_torch(seed = Config.random_seed)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:50:26.231658Z","iopub.execute_input":"2023-03-30T01:50:26.232234Z","iopub.status.idle":"2023-03-30T01:50:26.243177Z","shell.execute_reply.started":"2023-03-30T01:50:26.232195Z","shell.execute_reply":"2023-03-30T01:50:26.241561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'].to_pandas().hist(bins = Config.N_CLASS)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:51:41.866300Z","iopub.execute_input":"2023-03-30T01:51:41.866777Z","iopub.status.idle":"2023-03-30T01:51:42.620039Z","shell.execute_reply.started":"2023-03-30T01:51:41.866737Z","shell.execute_reply":"2023-03-30T01:51:42.618489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Before split:\")\nprint(\"train shape:\", train.shape)\n\ntrain_df, valid_df = train_test_split(train,\n                                      test_size=(1 - Config.split_fraction),\n                                      random_state=Config.random_seed)\n\nprint(\"\\nAfter split:\")\nprint(\"train shape:\", train_df.shape)\nprint(\"valid shape:\", valid_df.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:54:29.900362Z","iopub.execute_input":"2023-03-30T01:54:29.900893Z","iopub.status.idle":"2023-03-30T01:54:30.069280Z","shell.execute_reply.started":"2023-03-30T01:54:29.900850Z","shell.execute_reply":"2023-03-30T01:54:30.068088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitDataset(Dataset):\n    def __init__(self, df, X_col, y_col, augmentations = None):\n        self.features = df[X_col].to_numpy() / 255\n        self.targets = df[y_col].to_numpy().reshape((-1, 1))\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.features)\n    \n    def __getitem__(self, idx):\n        image = self.features[idx].reshape((1, 28, 28))\n        label = self.targets[idx]\n        \n        if self.augmentations is not None:\n            augmented = self.augmentations(image=image)\n            return torch.FloatTensor(augmented['image']), torch.FloatTensor(label)\n        else:\n            return torch.FloatTensor(image), torch.FloatTensor(label)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T01:58:56.822799Z","iopub.execute_input":"2023-03-30T01:58:56.823929Z","iopub.status.idle":"2023-03-30T01:58:56.833372Z","shell.execute_reply.started":"2023-03-30T01:58:56.823878Z","shell.execute_reply":"2023-03-30T01:58:56.831997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitInferenceDataset(Dataset):\n    def __init__(self, df, augmentations = None):\n        self.features = df.to_numpy() / 255\n        self.augmentations = augmentations\n        \n    def __len__(self):\n        return len(self.features)\n    \n    def __getitem__(self, idx):\n        image = self.features[idx].reshape((1, 28, 28))\n        return torch.FloatTensor(image)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T02:01:57.670808Z","iopub.execute_input":"2023-03-30T02:01:57.671337Z","iopub.status.idle":"2023-03-30T02:01:57.680463Z","shell.execute_reply.started":"2023-03-30T02:01:57.671292Z","shell.execute_reply":"2023-03-30T02:01:57.678764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = A.Compose([\n    A.Rotate(limit=40, p=Config.probability),\n    A.ShiftScaleRotate(rotate_limit=40, p=Config.probability),\n    A.Downscale(scale_min=0.7,\n                scale_max=0.7,\n                p=Config.probability,\n                interpolation=1)\n])\n\ntransform_valid = A.Compose([])","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:39.068542Z","iopub.execute_input":"2023-03-30T03:02:39.069086Z","iopub.status.idle":"2023-03-30T03:02:39.076799Z","shell.execute_reply.started":"2023-03-30T03:02:39.069047Z","shell.execute_reply":"2023-03-30T03:02:39.075006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_col = 'label'\nX_col = [c for c in train.columns if c != 'label']\n\nfig, ax = plt.subplots(5, 2, figsize=(6, 12))\nfig.suptitle(\"Checking some augmentations\", fontsize=18)\n\ntrain_dataset = DigitDataset(train, X_col, y_col, augmentations=None)\ntrain_dataset_aug = DigitDataset(train, X_col, y_col, augmentations=transform_train)\n\ntrain_dataloader = DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True)\ntrain_dataloader_aug = DataLoader(train_dataset_aug, batch_size=Config.BATCH_SIZE, shuffle=True)\n\nprint(\"Length train_dataset (td):\", len(train_dataset))\nprint(\"Length train_dataloader (tl):\", len(train_dataloader))\nprint(\"Batch Size:\", Config.BATCH_SIZE)\n\nbegin = 40\nend = begin + 5\n\nfor i in range(begin, end):\n    for j in range(0, 1):\n        image, label = train_dataset.__getitem__(begin + i)\n        image_aug, label_aug = train_dataset_aug.__getitem__(begin + i)\n        \n        label = str(label.item())\n        ax[i - begin, j].set_title(\"Label: \"+ label, color='red')\n        ax[i - begin, j].imshow(np.squeeze(image), cmap='gray')\n        ax[i - begin, j].axis('off')\n        \n        ax[i - begin, j+1].set_title(\"Aug Label: \"+ label, color='red')\n        ax[i - begin, j+1].imshow(np.squeeze(image_aug), cmap='gray')\n        ax[i - begin, j+1].axis('off')","metadata":{"execution":{"iopub.status.busy":"2023-03-30T02:15:09.570279Z","iopub.execute_input":"2023-03-30T02:15:09.570841Z","iopub.status.idle":"2023-03-30T02:15:10.685368Z","shell.execute_reply.started":"2023-03-30T02:15:09.570795Z","shell.execute_reply":"2023-03-30T02:15:10.683640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DigitModel(nn.Module):\n    def __init__(self):\n        super(DigitModel, self).__init__()\n        self.conv1 = nn.Conv2d(in_channels=1,\n                               out_channels=32,\n                               kernel_size=(5,5),\n                               padding=0)\n        self.actv = nn.LeakyReLU()\n        self.batchnorm1 = nn.BatchNorm2d(32)\n        self.maxpool = nn.MaxPool2d(kernel_size=(2,2))\n        self.dropout = nn.Dropout(0.25)\n        self.conv2 = nn.Conv2d(in_channels=32,\n                               out_channels=64,\n                               kernel_size=(5,5),\n                               padding=0)\n        self.batchnorm2 = nn.BatchNorm2d(64)\n        self.flatten = nn.Flatten()\n        self.fc1 = nn.Linear(64 * 4 * 4, 256)\n        self.batchnorm3 = nn.BatchNorm1d(256)\n        self.classifier = nn.Linear(256, Config.N_CLASS)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.actv(x)\n        x = self.batchnorm1(x)\n        x = self.maxpool(x)\n        \n        x = self.conv2(x)\n        x = self.actv(x)\n        x = self.batchnorm2(x)\n        x = self.maxpool(x)\n        \n        x = self.flatten(x)\n        \n        x = self.fc1(x)\n        x = self.batchnorm3(x)\n        \n        out = self.classifier(x)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2023-03-30T02:28:08.456468Z","iopub.execute_input":"2023-03-30T02:28:08.457265Z","iopub.status.idle":"2023-03-30T02:28:08.473708Z","shell.execute_reply.started":"2023-03-30T02:28:08.457206Z","shell.execute_reply":"2023-03-30T02:28:08.471799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_optimizer(model, lr = Config.learning_rate):\n    if Config.optimizer == 'adam':\n        optimizer = Adam(model.parameters(),\n                         lr=lr,\n                         weight_decay=Config.weight_decay,\n                         amsgrad=False)\n    elif Config.optimizer == 'adamw':\n        optimizer = AdamW(model.parameters(),\n                          lr=lr,\n                          weight_decay=Config.weight_decay)\n    elif Config.optimizer == 'rmsprop':\n        optimizer = RMSprop(model.parameters(),\n                          lr=lr,\n                          weight_decay=Config.l2_penalty,\n                          momentum=Config.rms_momentum)\n    else:\n        print(\"Optimizer is not defined\")\n    \n    return optimizer","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:19.978151Z","iopub.execute_input":"2023-03-30T03:02:19.979020Z","iopub.status.idle":"2023-03-30T03:02:19.988340Z","shell.execute_reply.started":"2023-03-30T03:02:19.978970Z","shell.execute_reply":"2023-03-30T03:02:19.986784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(DigitModel(), input_size=(1, 28, 28), batch_size=Config.BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:20.217030Z","iopub.execute_input":"2023-03-30T03:02:20.217462Z","iopub.status.idle":"2023-03-30T03:02:20.234177Z","shell.execute_reply.started":"2023-03-30T03:02:20.217426Z","shell.execute_reply":"2023-03-30T03:02:20.232559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_scheduler(optimizer):\n    if Config.scheduler == 'ReduceLROnPlateau':\n        scheduler = ReduceLROnPlateau(optimizer,\n                                      mode='max',\n                                      factor=Config.plateau_factor,\n                                      patience=Config.plateau_patience,\n                                      verbose=Config.verbose)\n    elif Config.scheduler == 'CosineAnnealingLR':\n        scheduler = CosineAnnealingLR(optimizer,\n                                      T_max=Config.cosine_T_max,\n                                      eta_min=Config.cosine_eta_min)\n    else:\n        print(\"LR scheduler is not defined\")\n        \n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:20.401050Z","iopub.execute_input":"2023-03-30T03:02:20.401446Z","iopub.status.idle":"2023-03-30T03:02:20.408363Z","shell.execute_reply.started":"2023-03-30T03:02:20.401413Z","shell.execute_reply":"2023-03-30T03:02:20.407022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_fn(train_dataloader,\n             model,\n             loss_fn,\n             optimizer,\n             device):\n    size = len(train_dataloader.dataset)\n    num_batches = len(train_dataloader)\n    \n    loss, correct = 0, 0\n    \n    model.train()\n    \n    for batch, (X, y) in enumerate(train_dataloader):\n        start = time.time()\n        device = torch.device(device)\n        X, y = X.to(device), y.to(device)\n        \n        optimizer.zero_grad()\n        pred = model(X)\n        loss = loss_fn(pred, y.long().squeeze())\n        \n        loss.backward()\n        optimizer.step()\n        \n        y_pred, y_true = torch.argmax(pred, axis=1), y.long().squeeze()\n        correct += (y_pred == y_true).type(torch.float).sum().item()\n        \n        end = time.time()\n        time_delta = np.round(end - start, 3)\n        \n        loss = np.round(loss.item(), 5)\n        \n    correct /= size\n    loss /= num_batches\n    print(f\"Train: accuracy: {(100 * correct):>0.2f}%, Avg loss: {loss:>5f}\\n\")\n    \n    return loss, correct","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:20.598539Z","iopub.execute_input":"2023-03-30T03:02:20.598984Z","iopub.status.idle":"2023-03-30T03:02:20.610016Z","shell.execute_reply.started":"2023-03-30T03:02:20.598948Z","shell.execute_reply":"2023-03-30T03:02:20.608627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_fn(valid_dataloader, model, loss_fn, device):\n    model.eval()\n    \n    size = len(valid_dataloader.dataset)\n    num_batches = len(valid_dataloader)\n    \n    loss, correct = 0, 0\n    \n    with torch.no_grad():\n        for batch, (X, y) in enumerate(valid_dataloader):\n            start = time.time()\n            \n            device = torch.device(device)\n            X, y = X.to(device), y.to(device)\n            \n            pred = model(X)\n            loss = loss_fn(pred, y.long().squeeze())\n            \n            y_pred, y_true = torch.argmax(pred, axis=1), y.long().squeeze()\n            \n            correct += (y_pred == y_true).type(torch.float).sum().item()\n            \n            end = time.time()\n            time_delta = np.round(end - start, 3)\n            \n            loss = np.round(loss.item(), 5)\n            \n        correct /= size\n        loss /= num_batches\n        \n        print(f\"Valid: Accuracy: {(100*correct):>0.2f}%, Avg loss: {loss:>5f}\\n\")\n        \n        return loss, correct","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:20.757645Z","iopub.execute_input":"2023-03-30T03:02:20.758798Z","iopub.status.idle":"2023-03-30T03:02:20.768520Z","shell.execute_reply.started":"2023-03-30T03:02:20.758753Z","shell.execute_reply":"2023-03-30T03:02:20.767281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start = time.time()\n\nloss_fn = nn.CrossEntropyLoss()\nmodel = DigitModel().to(device)\n\noptimizer = get_optimizer(model, lr=Config.learning_rate)\nscheduler = get_scheduler(optimizer)\n\ny_col = 'label'\nX_col = [c for c in train.columns if c != 'label']\n\ntrain_dataset = DigitDataset(train_df,\n                             X_col,\n                             y_col,\n                             augmentations=transform_train)\nvalid_dataset = DigitDataset(valid_df,\n                             X_col,\n                             y_col,\n                             augmentations=transform_valid)\n\ntrain_dataloader = DataLoader(train_dataset,\n                              batch_size=Config.BATCH_SIZE,\n                              shuffle=Config.shuffle)\nvalid_dataloader = DataLoader(valid_dataset,\n                              batch_size=Config.BATCH_SIZE,\n                              shuffle=Config.shuffle)\n\ntrain_loss_history = []\ntrain_acc_history = []\nvalid_loss_history = []\nvalid_acc_history = []\nLR_history = []\n\nbest_loss = np.inf\nbest_epoch_loss = 0\nbest_acc = 0\nbest_epoch_acc = 0\n\nprint(\"Starting Training...\\n\")\n\nstart_train_time = time.time()\n\nfor epoch in range(Config.EPOCHS):\n    print(f\"Epoch {epoch+1}\")\n    start_epoch_time = time.time()\n    \n    train_loss, train_acc = train_fn(train_dataloader,\n                                     model,\n                                     loss_fn,\n                                     optimizer,\n                                     device)\n    train_loss_history.append(train_loss)\n    train_acc_history.append(train_acc)\n    \n    valid_loss, valid_acc = valid_fn(valid_dataloader,\n                                     model,\n                                     loss_fn,\n                                     device)\n    valid_loss_history.append(valid_loss)\n    valid_acc_history.append(valid_acc)\n    \n    if isinstance(scheduler, ReduceLROnPlateau):\n        scheduler.step(valid_loss)\n    elif isinstance(schedule, CosineAnnealingLR):\n        scheduler.step()\n        \n    for param_group in optimizer.param_groups:\n        LR_history.append(param_group['lr'])\n        \n    if valid_loss < best_loss:\n        best_epoch_loss = epoch + 1\n        best_loss = valid_loss\n        if Config.save_best_loss:\n            torch.save(model.state_dict(),\n                       OUTPUT_PATH + f'DigitModel_{best_epoch_loss}.pth')\n    \n    if valid_acc > best_acc:\n        best_epoch_acc = epoch + 1\n        best_acc = valid_acc\n        if Config.save_best_accuracy:\n            torch.save(model.state_dict(),\n                       OUTPUT_PATH + f'DigitModel_{best_epoch_acc}.pth')\n            \n    end_epoch_time = time.time()\n    time_delta = np.round(end_epoch_time - start_epoch_time, 3)\n    print(\"\\n\\nEpoch Elapsed Time: {} s\".format(time_delta))\n    \nend_train_time = time.time()\nprint(\"\\n\\nTotal Elapsed Time: {} min\".format(np.round((end_train_time - start_train_time) / 60, 3)))\nprint(\"Done!\")","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:02:44.094585Z","iopub.execute_input":"2023-03-30T03:02:44.095021Z","iopub.status.idle":"2023-03-30T03:23:10.302400Z","shell.execute_reply.started":"2023-03-30T03:02:44.094986Z","shell.execute_reply":"2023-03-30T03:23:10.300771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best loss:\", best_loss)\nprint(\"Best epoch (loss criteria):\", best_epoch_loss)\nprint()\nprint(\"Best accuracy:\", best_acc)\nprint(\"Best epoch (accuracy criteria):\", best_epoch_acc)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:41:23.591946Z","iopub.execute_input":"2023-03-30T03:41:23.592358Z","iopub.status.idle":"2023-03-30T03:41:23.599416Z","shell.execute_reply.started":"2023-03-30T03:41:23.592324Z","shell.execute_reply":"2023-03-30T03:41:23.598201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(18, 8))\nfig.suptitle(\"Epoch Results\", fontsize = 18)\n\nabscissa = np.arange(1, Config.EPOCHS + 1, 1)\nif Config.EPOCHS <= 20:\n    x_ticks = np.arange(1, Config.EPOCHS + 1, 1)\nelse:\n    x_ticks = np.arange(1, Config.EPOCHS + 1, int(Config.EPOCHS / 20) + 1)\n    \nax1 = plt.subplot(1, 2, 1)\nax1.plot(abscissa, train_loss_history, label=\"Training\", color=\"black\")\nax1.plot(abscissa, valid_loss_history, label=\"Validation\", color=\"red\")\nplt.xticks(x_ticks)\nplt.axhline(0, linestyle='dashed', color='grey')\nplt.axvline(best_epoch_loss,\n            linestyle='dashed',\n            color='blue',\n            label=\"Best val loss: ep \" + str(best_epoch_loss))\nplt.axvline(best_epoch_acc,\n            linestyle='dashed',\n            color='green',\n            label='Best val acc: ep ' + str(best_epoch_acc))\nplt.title(\"Loss\")\nax1.legend(frameon=False)\n\nax2 = plt.subplot(1, 2, 2)\nax2.plot(abscissa, train_acc_history, label=\"Training\", color=\"black\")\nax2.plot(abscissa, valid_acc_history, label=\"Validation\", color=\"red\")\nplt.xticks(x_ticks)\nplt.axhline(0.99, linestyle=\"dashed\", color=\"grey\")\nplt.axvline(best_epoch_loss,\n            linestyle=\"dashed\",\n            color=\"blue\",\n            label=\"Best val loss: ep \" + str(best_epoch_loss))\nplt.axvline(best_epoch_acc,\n            linestyle=\"dashed\",\n            color=\"green\",\n            label=\"Best val acc: ep \" + str(best_epoch_acc))\nplt.title(\"Accuracy\")\nax2.legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:51:50.020830Z","iopub.execute_input":"2023-03-30T03:51:50.021230Z","iopub.status.idle":"2023-03-30T03:51:50.591987Z","shell.execute_reply.started":"2023-03-30T03:51:50.021198Z","shell.execute_reply":"2023-03-30T03:51:50.590757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(14, 8))\n\nabscissa = np.arange(1, Config.EPOCHS + 1, 1)\n\nif Config.EPOCHS <= 20:\n    x_ticks = np.arange(1, Config.EPOCHS + 1, 1)\nelse:\n    x_ticks = np.arange(1, Config.EPOCHS + 1, int(Config.EPOCHS / 20) + 1)\n    \nplt.plot(abscissa, LR_history, label=\"LR\", color=\"orange\")\nplt.xticks(x_ticks)\nplt.axhline(Config.learning_rate, linestyle=\"dashed\", color=\"grey\")\nplt.axhline(0, linestyle=\"dashed\", color=\"grey\")\nplt.axvline(best_epoch_loss,\n            linestyle=\"dashed\",\n            color=\"blue\",\n            label=\"Best val loss: ep \" + str(best_epoch_loss))\nplt.axvline(best_epoch_acc,\n            linestyle=\"dashed\",\n            color=\"green\",\n            label=\"Best val acc: ep \" + str(best_epoch_acc))\nplt.title(f\"Learning Rate vs Epochs: {Config.scheduler}\",\n          fontsize=16,\n          color=\"orange\")\nplt.legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T03:58:11.257374Z","iopub.execute_input":"2023-03-30T03:58:11.257836Z","iopub.status.idle":"2023-03-30T03:58:11.691532Z","shell.execute_reply.started":"2023-03-30T03:58:11.257798Z","shell.execute_reply":"2023-03-30T03:58:11.689857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(x):\n    return np.exp(x) / np.sum(np.exp(x), axis=1)[:, None]\n\ndef inference(test_dataloader, model):\n    model.eval()\n    \n    predictions = []\n    \n    size = len(test_dataloader.dataset)\n    num_batches = len(test_dataloader)\n    \n    model = DigitModel().to(device)\n    \n    if Config.save_best_loss:\n        model.load_state_dict(\n            torch.load(OUTPUT_PATH + f'DigitModel_{best_epoch_loss}.pth')\n        )\n    else:\n        model.load_state_dict(\n            torch.load(OUTPUT_PATH + f'DigitModel_{best_epoch_acc}.pth')\n        )\n        \n    with torch.no_grad():\n        for batch, X in enumerate(test_dataloader):\n            start = time.time()\n            \n            X = X.to(device)\n            \n            pred = model(X)\n            y_pred = softmax(pred.detach().cpu().numpy())\n            y_pred = np.argmax(y_pred, axis = 1)\n            \n            predictions.append(y_pred)\n            \n            end = time.time()\n            time_delta = np.round(end - start, 5)\n            \n    test_predictions = np.concatenate(predictions, axis=0)\n    return test_predictions","metadata":{"execution":{"iopub.status.busy":"2023-03-30T04:12:33.563134Z","iopub.execute_input":"2023-03-30T04:12:33.563556Z","iopub.status.idle":"2023-03-30T04:12:33.576233Z","shell.execute_reply.started":"2023-03-30T04:12:33.563522Z","shell.execute_reply":"2023-03-30T04:12:33.575009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = DigitInferenceDataset(test, augmentations=None)\ntest_dataloader = DataLoader(test_dataset,\n                             batch_size=Config.BATCH_SIZE,\n                             shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T04:15:01.578611Z","iopub.execute_input":"2023-03-30T04:15:01.580024Z","iopub.status.idle":"2023-03-30T04:15:01.810764Z","shell.execute_reply.started":"2023-03-30T04:15:01.579963Z","shell.execute_reply":"2023-03-30T04:15:01.809259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = inference(test_dataloader, model)\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-03-30T04:15:23.011138Z","iopub.execute_input":"2023-03-30T04:15:23.011568Z","iopub.status.idle":"2023-03-30T04:15:33.196783Z","shell.execute_reply.started":"2023-03-30T04:15:23.011532Z","shell.execute_reply":"2023-03-30T04:15:33.195750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\nsubmission = pd.read_csv(INPUT_PATH + 'sample_submission.csv')\nsubmission[\"Label\"] = predictions\n\nsubmission.to_csv(OUTPUT_PATH + 'submission.csv', index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T04:16:31.422103Z","iopub.execute_input":"2023-03-30T04:16:31.422529Z","iopub.status.idle":"2023-03-30T04:16:31.516386Z","shell.execute_reply.started":"2023-03-30T04:16:31.422491Z","shell.execute_reply":"2023-03-30T04:16:31.515020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}