{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-10-30T12:50:02.828667Z","iopub.execute_input":"2022-10-30T12:50:02.829021Z","iopub.status.idle":"2022-10-30T12:50:02.842022Z","shell.execute_reply.started":"2022-10-30T12:50:02.828985Z","shell.execute_reply":"2022-10-30T12:50:02.838713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nimport torch\n\n\nclass ConvLayer(nn.Module):\n    def __init__(self, in_channels, out_channels, pool=False, kernel_size=3, dropout=0.15, act=\"swish\"):\n        super().__init__()\n        # output_dim = (input_dim + 2 *padding - dilation * (kernel_size -1) - 1) / (stride) +1\n        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, padding_mode=\"reflect\", padding=kernel_size//2)\n        if act == \"swish\":\n            self.act = nn.SiLU()\n        elif act == \"RELU\":\n            self.act = nn.ReLU()\n        elif act == \"leaky\":\n            self.act = nn.LeakyReLU(0.1)\n        else:\n            self.act = nn.ELU()\n        self.batch_norm = nn.BatchNorm2d(out_channels, momentum=0.9)\n        self.dropout = nn.Dropout2d(dropout)\n        self.pool = nn.MaxPool2d(2,2) if pool else nn.Identity()\n\n    def forward(self,x):\n        x = self.conv(x)\n        x = self.act(x)\n        x = self.batch_norm(x)\n        x = self.dropout(x)\n        x = self.pool(x)\n        return x\n\n\nclass Net(nn.Module):\n    def __init__(self, use_log_softmax=False):\n        super().__init__()\n        # self.convs = nn.Sequential(\n        #         ConvLayer(1,16),\n        #         ConvLayer(16,32, dropout=0.2),\n        #         ConvLayer(32, 64, dropout=0.2),\n        #         ConvLayer(64, 128, pool=True, dropout=0.2),\n        #         ConvLayer(128, 128, pool=False, dropout=0.1),\n        #         ConvLayer(128, 64, pool=True),\n        #         ConvLayer(64, 32),\n        #         ConvLayer(32, 16, dropout=0.1),\n        #         # ConvLayer(16, 16)\n        #         )\n\n\n        self.conv1 = nn.Sequential(\n                ConvLayer(1, 64,dropout=0),\n                ConvLayer(64, 128, dropout=0),\n                ConvLayer(128, 128, pool=True, dropout=0.25),\n                )\n        \n        self.conv2_1x1 = nn.Sequential(\n                ConvLayer(128, 128,dropout=0,kernel_size=1),\n                ConvLayer(128, 128, pool=False , dropout=0,kernel_size=1),\n                ConvLayer(128, 128, pool=False , dropout=0,kernel_size=1),\n                )\n\n        self.conv2_3x3 = nn.Sequential(\n                ConvLayer(128, 128,dropout=0),\n                ConvLayer(128, 128, dropout=0),\n                ConvLayer(128, 128, pool=False , dropout=0),\n                )\n        self.conv2_5x5 = nn.Sequential(\n                ConvLayer(128, 128, kernel_size=5, dropout=0,),\n                ConvLayer(128, 128, dropout=0, kernel_size=5),\n                ConvLayer(128, 128, pool=False , dropout=0, kernel_size=5),\n                )\n        \n        # self.conv25 = nn.Identity() # ConvLayer(256,256,kernel_size=1)\n        self.conv25 = ConvLayer(128*3,256,kernel_size=1)\n\n        self.conv3_1x1 = nn.Sequential(\n                ConvLayer(256, 256,dropout=0,kernel_size=1),\n                )\n        \n        self.conv3_3x3 = nn.Sequential(\n                ConvLayer(256, 256, dropout=0,),\n#                 ConvLayer(256, 256, dropout=0),\n                )\n        self.conv3_5x5 = nn.Sequential(\n                ConvLayer(256, 256, kernel_size=5, dropout=0),\n#                 ConvLayer(256, 256, dropout=0, kernel_size=5),\n                )\n        \n        self.conv35 = ConvLayer(256*3,512,kernel_size=3,dropout=0)\n        \n#         self.conv4 = ConvLayer(256,256)\n\n\n        self.pool = nn.MaxPool2d((2,2))\n\n        self.dropout = nn.Dropout(0.25)\n        \n        self.act = nn.SiLU()\n\n        self.dense_layers = nn.Sequential(\n                nn.Linear(512 * 3 * 3, 256),\n                self.act,\n                nn.BatchNorm1d(256, momentum=0.5),\n                nn.Dropout(0.2),\n                nn.Linear(256, 10),\n                )\n        self.log_softmax = nn.LogSoftmax(dim=1) if use_log_softmax else nn.Identity()\n\n    def forward(self, x):\n        block1 = self.conv1(x)\n        \n        block2_1x1_out = self.conv2_1x1(block1)\n        block2_3x3_out = self.conv2_3x3(block1)\n        block2_5x5_out = self.conv2_5x5(block1)\n        \n        block2_1x1_out += block1\n        block2_3x3_out += block1\n        block2_5x5_out += block1\n        \n        block2 = torch.cat((block2_1x1_out,block2_3x3_out,block2_5x5_out),dim=1)\n        block2 = self.conv25(block2)\n        \n        block2_out = self.dropout(self.pool(block2))\n        \n        block3_1x1_out = self.conv3_1x1(block2_out)\n        block3_3x3_out = self.conv3_3x3(block2_out)\n        block3_5x5_out = self.conv3_5x5(block2_out)\n        \n        block3_1x1_out += block2_out\n        block3_3x3_out += block2_out\n        block3_5x5_out += block2_out\n        \n        block3 = torch.cat((block3_1x1_out,block3_3x3_out,block3_5x5_out),dim=1)\n        block3 = self.conv35(block3)\n        block3_out = self.dropout(self.pool(block3))\n\n        x = torch.flatten(block3_out, 1) # flatten all dimensions except batch\n        \n        x = self.dense_layers(x)\n        return self.log_softmax(x)","metadata":{"execution":{"iopub.status.busy":"2022-10-30T12:50:02.844467Z","iopub.execute_input":"2022-10-30T12:50:02.845124Z","iopub.status.idle":"2022-10-30T12:50:02.872397Z","shell.execute_reply.started":"2022-10-30T12:50:02.845083Z","shell.execute_reply":"2022-10-30T12:50:02.871472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### import numpy as np\nimport torch\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import DataLoader\nfrom torchvision import transforms\n# from NN import Net\n\nimport random\n\nimport tqdm\n\n\n\nmode = \"none\"\n# mode = \"online\"\nload_checkpoint = True\nif mode != \"none\":\n    import neptune.new as neptune\nif mode == \"debug\":\n    run = neptune.init_run(mode=\"debug\")\nelif mode == \"offline\":\n    neptune.init(mode=\"offline\")\nelif mode == \"online\":\n    run = neptune.init(\n            project=\"julian-wyatt/Kannada-MNIST\",\n            api_token=\"eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiJlNDA3YjhmZS1jNjM0LTRhMzYtYTY0Yi0wMGMyYjMzMTI3OGYifQ==\",\n            source_files=[\"main.py\", \"NN.py\"]\n        )\n\n\nEPOCHS = 0 if load_checkpoint else 50\ndef train(model, criterion, optimiser, iterator):\n    import time\n    start = time.time()\n    losses = 0\n    accuracies = 0\n    for i,batch in enumerate(iterator):\n        x,y = batch\n        x,y = x.to(device), y.to(device)\n\n        prediction = model(x)\n        loss = criterion(prediction,y)\n\n        loss.backward()\n        optimiser.step()\n        with torch.no_grad():\n            acc = calculate_accuracy(torch.argmax(prediction,dim=1),y)\n            accuracies += acc.item()\n            losses += loss.item()\n            if mode != \"none\":\n                run[\"metrics/train/accuracy\"].log(acc.item())\n                run[\"metrics/train/loss\"].log(loss.item())\n    if mode != \"none\":\n        run[\"metrics/train/mean_accuracy\"].log(accuracies/len(iterator))\n        run[\"metrics/train/mean_loss\"].log(losses/len(iterator))\n    return losses/len(iterator), accuracies/len(iterator)\n\n\n\ndef evaluate(model, criterion, iterator):\n    losses = 0\n    accuracies = 0\n\n    f1_scores = [[] for _ in range(10)]\n    for i,batch in enumerate(iterator):\n        x,y=batch\n        x, y = x.to(device), y.to(device)\n        prediction = model(x)\n        with torch.no_grad():\n            loss = criterion(prediction,y)\n            losses += loss\n            ACC, Pres, Rec, f1 = metrics_calculations(prediction,y)\n\n            for i in range(10):\n                f1_scores[i].append(f1[i].item())\n                # run[f\"metrics/test/{i+1}/Precision\"].log(Pres[i].item())\n                # run[f\"metrics/test/{i+1}/Recall\"].log(Rec[i].item())\n                # run[f\"metrics/test/{i+1}/f1\"].log(f1[i].item())\n        accuracies += ACC.item()\n        if mode != \"none\":\n            run[\"metrics/test/accuracy\"].log(ACC.item())\n            run[\"metrics/test/loss\"].log(loss.item())\n\n    # f1's per batch\n    total_f1 = []\n    for i,arr in enumerate(f1_scores):\n        mean_f1 = sum(arr)/len(arr)\n        total_f1.append(mean_f1)\n        if mode != \"none\":\n            run[f\"metrics/test/{i}/f1\"].log(mean_f1)\n    # mean f1 per batch over all digits\n    if mode != \"none\":\n        run[\"metrics/test/mean_f1\"].log(sum(total_f1)/len(total_f1))\n\n        run[\"metrics/test/mean_accuracy\"].log(accuracies / len(iterator))\n        run[\"metrics/test/mean_loss\"].log(losses / len(iterator))\n    # Add neptune logging here\n    return losses/len(iterator), accuracies / len(iterator)\n\ndef training_loop(model, training_set, test_set, optimiser, criterion, epochs, scheduler, best_model:Net):\n    best_loss=100\n    epochs_without_improvement=0\n    early_stopping_total = 15\n    early_stopping_eps = 0\n    for epoch in range(epochs):\n        \n        model.train()\n        train_loss, train_accuracy = train(model,criterion,optimiser,training_set)\n        model.eval()\n        val_loss, val_accuracy = evaluate(model,criterion,test_set)\n        scheduler.step(val_loss)\n        print(f\"EPOCH {epoch:02d} - Train Loss {train_loss:.4f} & Acc: {train_accuracy:.4f}, Val Loss {val_loss:.4f} & Acc: {val_accuracy:.4f}\")\n        if val_loss + early_stopping_eps < best_loss:\n            best_loss=val_loss\n            best_model.load_state_dict(model.state_dict())\n            epochs_without_improvement=0\n        else:\n            epochs_without_improvement+=1\n        if epochs_without_improvement==early_stopping_total:\n            print(\"Early stopping!\")\n            break\n          \n        \n\nclass csvImageDataset(torch.utils.data.Dataset):\n    def __init__(self, df, train=False):\n        self.df = df\n        self.train = train\n        if self.train:\n            self.transform = transforms.Compose(\n                    [transforms.ToPILImage(),\n                     transforms.RandomAffine(\n                         10, translate=(0.15, 0.15), scale=(0.7, 1.3), shear=0.1,\n                         interpolation=transforms.InterpolationMode.BILINEAR\n                         ),\n                     transforms.ToTensor(),\n                     transforms.RandomErasing(p=0.2, scale=(0.02, 0.2)),\n                     transforms.Normalize((0.5), (0.5)),\n                     \n                     ])\n                    \n        else:\n            self.transform = transforms.Compose(\n                    [transforms.ToPILImage(),\n                     transforms.ToTensor(),\n                     transforms.Normalize((0.5), (0.5))\n                     ]\n                    )\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index].tolist()\n        image = np.array(row[1:],dtype=np.uint8)\n        image = image.reshape((28, 28, 1))\n#         if self.train and random.random() > 0.8:\n#             image = self.transform2(image)\n#         else:\n        image = self.transform(image)\n        label = torch.tensor(row[0])\n        return image, label\n\n\n# make accuracy, recall, precision, f1-score functions\ndef calculate_accuracy(y_hat,y):\n    TP_overall = torch.sum(y_hat == y)\n    ACC = TP_overall / y_hat.shape[0]\n    return ACC\n\ndef metrics_calculations(y_hat: torch.Tensor,y:torch.Tensor):\n    with torch.no_grad():\n        # custom implementation for classification metrics over 10 classes\n        y_hat = torch.argmax(y_hat,dim=1)\n\n        ACC = calculate_accuracy(y_hat,y)\n\n        Precisions = [] # TP / (FP + TP)\n        Recalls = [] # TP / (FN + TP)\n        f1scores = [] # 2 P*R / (P+R)\n\n        for i in range(10):\n            # Best way to remember FP & FN\n            # false positive is incorrectly classify as 0 ie 0 is wrong\n            # false negative is incorrectly classify a 0 as not 0\n            relevant_y = y==i\n            relevant_y_hat = y_hat == i\n\n            TP = torch.sum(y_hat[relevant_y] == y[relevant_y])\n            FP = torch.sum(y_hat[relevant_y_hat] != y[relevant_y_hat])\n            FN = torch.sum(y_hat[relevant_y] != y[relevant_y])\n            if FP + TP == 0:\n                Precisions.append(TP*0)\n            else:\n                Precisions.append(TP/(FP+TP))\n            if FN+TP == 0:\n                Recalls.append(TP*0)\n            else:\n                Recalls.append(TP/(FN+TP))\n            if Recalls[-1] == 0 and Precisions[-1] == 0:\n                f1scores.append(TP*0)\n            else:\n                f1scores.append(2*Precisions[-1]*Recalls[-1]/(Precisions[-1]+Recalls[-1]))\n\n        return ACC, Precisions, Recalls, f1scores\n\ndef make_submission(classifier,test_set):\n    df = pd.read_csv(f\"{kaggle_prefix}Kannada-MNIST/sample_submission.csv\")\n    classifier.eval()\n    predictions = []\n    for i,batch in enumerate(test_set):\n        x,y = batch\n        x, y = x.to(device), y.to(device)\n        out = classifier(x)\n        out = torch.argmax(out,dim=1).squeeze()\n        predictions += list(out.cpu().numpy())\n    df[\"label\"] = predictions\n    df.to_csv(\"submission.csv\",index=False)\n    if mode != \"none\":\n        run[\"submission\"].upload_files(\"submission.csv\")\n\n\n\nif __name__ == \"__main__\":\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    # init model, iterator, optimiser\n    kaggle_prefix = \"/kaggle/input/\"\n    df_train = pd.read_csv(f\"{kaggle_prefix}Kannada-MNIST/train.csv\")\n    df_test = pd.read_csv(f\"{kaggle_prefix}Kannada-MNIST/test.csv\")\n    df_test_OOD = pd.read_csv(f\"{kaggle_prefix}Kannada-MNIST/Dig-MNIST.csv\")\n\n    ratio = 0.8\n    df_train_subset, df_val_subset = df_train.iloc[:int(len(df_train)*ratio)], df_train.iloc[int(len(df_train)*ratio):]\n\n    train_dataset = csvImageDataset(df_train, train=True)\n    val_dataset = csvImageDataset(df_val_subset)\n    test_dataset = csvImageDataset(df_test)\n    Batch_size = 256\n    train_dataloader = DataLoader(train_dataset, batch_size=Batch_size, shuffle=True)\n    val_dataloader = DataLoader(val_dataset, batch_size=Batch_size, shuffle=False)\n    test_dataloader = DataLoader(test_dataset, batch_size=Batch_size, shuffle=False)\n\n    classifier = Net()\n    best_model = Net()\n    best_model.load_state_dict(torch.load(f\"{kaggle_prefix}checkpoint-model-mnist-5/saved_model-5.pt\"))\n    print(classifier)\n    classifier = classifier.to(device)\n    # optimiser = torch.optim.AdamW(classifier.parameters(),lr=0.0001, weight_decay=1e-6)\n    optimiser = torch.optim.Adam(classifier.parameters(),lr=0.00001 if not load_checkpoint else 5e-9)\n#     optimiser = torch.optim.SGD(classifier.parameters(), lr=0.001)\n    criterion = torch.nn.CrossEntropyLoss()\n    # criterion = torch.nn.NLLLoss()\n\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimiser, 'min',factor=0.2, patience=2, verbose=True, \n                                                           min_lr=1e-7, cooldown=1,)\n    \n#     for param in classifier.parameters():\n#         param.requires_grad = False\n#     for param in classifier.dense_layers.parameters():\n#         param.requires_grad = True\n\n    training_loop(classifier,train_dataloader,val_dataloader,optimiser,criterion,EPOCHS, scheduler, best_model)\n\n    make_submission(best_model.to(device),test_dataloader)\n#     make_submission(classifier,test_dataloader)\n    torch.save(best_model.state_dict(),\"./saved_model.pt\")\n\n    # import torchvision\n\n    # for i,batch in enumerate(train_dataloader):\n    #     x,y = batch\n    #     print(i, y[0], x.shape)\n    #     plt.imshow(torchvision.utils.make_grid((x+1)/2, nrow=8, pad_value=5).cpu().data.permute(0, 2, 1).contiguous().permute(2, 1, 0),cmap=\"gray\")\n    #     plt.title(f\"TRAIN Class: {y[0]}\")\n    #     plt.show()\n    #     if i > 4:\n    #         break\n    # for i,batch in enumerate(val_dataloader):\n    #     x,y = batch\n    #     print(i, y[0], x.shape)\n    #     plt.imshow(torchvision.utils.make_grid((x+1)/2, nrow=8, pad_value=5).cpu().data.permute(0, 2, 1).contiguous().permute(2, 1, 0),cmap=\"gray\")\n    #     plt.title(f\"TEST Class: {y[0]}\")\n    #     plt.show()\n    #     if i > 4:\n    #         break\n    if mode != \"none\":\n        run.stop()","metadata":{"execution":{"iopub.status.busy":"2022-10-30T12:50:03.081614Z","iopub.execute_input":"2022-10-30T12:50:03.081919Z","iopub.status.idle":"2022-10-30T12:52:12.695978Z","shell.execute_reply.started":"2022-10-30T12:50:03.081892Z","shell.execute_reply":"2022-10-30T12:52:12.695126Z"},"trusted":true},"execution_count":null,"outputs":[]}]}