{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:03.061372Z","iopub.execute_input":"2022-11-04T18:14:03.061882Z","iopub.status.idle":"2022-11-04T18:14:03.073298Z","shell.execute_reply.started":"2022-11-04T18:14:03.061827Z","shell.execute_reply":"2022-11-04T18:14:03.072216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading Data","metadata":{}},{"cell_type":"code","source":"def load_data(filename, labels_present=True):\n\n    data = pd.read_csv(filename)\n    \n    # TODO - Currently misses first column if uses on data without label column\n\n    if labels_present:\n        \n        X = data.iloc[:,1:].to_numpy().astype(\"float32\").reshape(-1,28,28)/255.0\n        print(\"Training images shape:\", X.shape)\n        \n        Y = data[\"label\"].to_numpy().astype(\"float32\").reshape(-1,)\n        print(\"Training labels shape:\", Y.shape)\n        \n        return X, Y\n    \n    else:\n        \n        X = data.iloc[:,1:].to_numpy().astype(\"float32\").reshape(-1,28,28)/255.0\n        print(\"Test images shape:\", X.shape)\n        \n        ids = data[\"id\"].to_numpy().astype(\"int\").reshape(-1,)\n        print(\"IDs shape:\", ids.shape)\n        \n        return X, ids","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:03.080703Z","iopub.execute_input":"2022-11-04T18:14:03.081628Z","iopub.status.idle":"2022-11-04T18:14:03.090172Z","shell.execute_reply.started":"2022-11-04T18:14:03.081403Z","shell.execute_reply":"2022-11-04T18:14:03.089195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = \"../input/Kannada-MNIST/\"\n\nX, Y = load_data(data_dir + \"train.csv\")\n","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:03.091746Z","iopub.execute_input":"2022-11-04T18:14:03.092156Z","iopub.status.idle":"2022-11-04T18:14:06.139927Z","shell.execute_reply.started":"2022-11-04T18:14:03.092108Z","shell.execute_reply":"2022-11-04T18:14:06.138831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X[5])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.143139Z","iopub.execute_input":"2022-11-04T18:14:06.143827Z","iopub.status.idle":"2022-11-04T18:14:06.338756Z","shell.execute_reply.started":"2022-11-04T18:14:06.143785Z","shell.execute_reply":"2022-11-04T18:14:06.337882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(np.unique(Y))","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.341103Z","iopub.execute_input":"2022-11-04T18:14:06.341479Z","iopub.status.idle":"2022-11-04T18:14:06.351035Z","shell.execute_reply.started":"2022-11-04T18:14:06.341442Z","shell.execute_reply":"2022-11-04T18:14:06.350006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn as nn\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.352880Z","iopub.execute_input":"2022-11-04T18:14:06.353304Z","iopub.status.idle":"2022-11-04T18:14:06.927010Z","shell.execute_reply.started":"2022-11-04T18:14:06.353263Z","shell.execute_reply":"2022-11-04T18:14:06.926035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions\n\nCould remove cuda's, as hard to test","metadata":{}},{"cell_type":"code","source":" \nfrom torch.autograd import Variable\n\ndef cuda(v):\n    if torch.cuda.is_available():\n        return v.cuda()\n    return v\ndef toTensor(v,dtype = torch.float,requires_grad = False):       \n    return cuda(Variable(torch.tensor(v)).type(dtype).requires_grad_(requires_grad))\ndef toNumpy(v):\n    if torch.cuda.is_available():\n        return v.detach().cpu().numpy()\n    return v.detach().numpy()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.928399Z","iopub.execute_input":"2022-11-04T18:14:06.929500Z","iopub.status.idle":"2022-11-04T18:14:06.938241Z","shell.execute_reply.started":"2022-11-04T18:14:06.929450Z","shell.execute_reply":"2022-11-04T18:14:06.937024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loaders","metadata":{}},{"cell_type":"code","source":"import random\nimport sys\n\n# A dataset class that can be used by PyTorch data loaders\nclass CustomDataset(Dataset):\n    \n    def __init__(self, X, Y, num_classes=1, aug_transform=None):\n\n        self.X = X\n        # Convert labels to a tensor for training\n        # self.Y = nn.functional.one_hot(toTensor(Y).long(), num_classes=num_classes)\n        self.Y = toTensor(Y).long()\n        \n        X, Y = None, None\n\n        self.xi_preprocessing_transform = transforms.Compose([\n                                            transforms.ToTensor(),\n                                          ])\n        self.yi_preprocessing_transform = None\n        self.aug_transform = aug_transform\n        if self.aug_transform is not None:\n            print(\"[WARNING]: Dataset loaded with augmentations set, make sure this is training only\")\n        \n        self.nitems = self.X.shape[0]\n        \n        self.xi = None\n        self.yi = None\n        \n    def __getitem__(self, index):\n\n        X, Y = None, None # Avoiding accidently using X and Y instead of self.X\n\n        xi = self.X[index]\n        yi = self.Y[index]\n        \n        xi_final = xi\n        yi_final = yi\n        \n        seed = random.randrange(sys.maxsize) #get a random seed so that we can reproducibly do the transformations\n\n        # First augment the data, if we have some augmentations to apply\n        if self.aug_transform is not None:\n            random.seed(seed)\n\n            trans_dict = self.aug_transform(image=xi)\n            # Apply to the images before doing standard transforms\n            xi = trans_dict[\"image\"]\n\n        # Convert examples to a tensor\n        if self.xi_preprocessing_transform is not None:\n            random.seed(seed) # apply this seed to img transforms\n            xi_final = self.xi_preprocessing_transform(xi)\n\n        return xi_final,  yi_final\n    \n    def __len__(self):\n        return self.nitems","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.939757Z","iopub.execute_input":"2022-11-04T18:14:06.940746Z","iopub.status.idle":"2022-11-04T18:14:06.952902Z","shell.execute_reply.started":"2022-11-04T18:14:06.940708Z","shell.execute_reply":"2022-11-04T18:14:06.951775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Zoo","metadata":{}},{"cell_type":"markdown","source":"## BasicCNN","metadata":{}},{"cell_type":"code","source":"# Convolutional neural network (two convolutional layers)\nclass ConvNet(nn.Module):\n    # num_classes is the number of COUNTS to return\n    def __init__(self, in_channels=1, num_classes=10):\n        super(ConvNet, self).__init__()\n\n        self.num_classes = num_classes\n\n        self.name = \"ConvNet\"\n\n        # Layer 1 variables:\n        layer1_kernel_num = 64\n        self.layer1 = nn.Sequential(\n            nn.Conv2d(in_channels, layer1_kernel_num, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(layer1_kernel_num), # Should match output size of layer\n            nn.LeakyReLU(negative_slope=0.1),\n            # nn.MaxPool2d(kernel_size=2, stride=2)\n            )\n\n        # Layer 2 variables:\n        layer2_kernel_num = 64\n        self.layer2 = nn.Sequential(\n            nn.Conv2d(layer1_kernel_num, layer2_kernel_num, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(layer2_kernel_num),\n            nn.LeakyReLU(negative_slope=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2))\n        \n        layer3_kernel_num = 128\n        self.layer3 = nn.Sequential(\n            nn.Conv2d(layer2_kernel_num, layer3_kernel_num, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(layer3_kernel_num),\n            nn.LeakyReLU(negative_slope=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n            )\n        \n        layer4_kernel_num = 128\n        self.layer4 = nn.Sequential(\n            nn.Conv2d(layer3_kernel_num, layer4_kernel_num, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(layer4_kernel_num),\n            nn.LeakyReLU(negative_slope=0.1),\n            nn.MaxPool2d(kernel_size=2, stride=2)\n            )\n\n        self.fc1 = nn.Sequential(\n            nn.Linear(1152, 256),\n            nn.Dropout(p=0.2)\n            )\n        \n        # Try sequential + dropout\n        self.fc2= nn.Linear(1152, num_classes)\n\n        self.output_layer = nn.Softmax(dim=1)\n\n        print(\"Model initialised.\")\n\n    # x: numpy array with shape (256,256,3)\n    def forward(self, x):\n\n        out = self.layer1(x)\n\n        out = self.layer2(out)\n        \n        out = self.layer3(out)\n        \n        out = self.layer4(out)\n\n        # Flatten output of Conv part\n        out = out.reshape(out.size(0), -1)\n\n#         print(out.size())\n        \n#         out = self.fc1(out)\n        out = self.fc2(out)\n        out = self.output_layer(out)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.955406Z","iopub.execute_input":"2022-11-04T18:14:06.956097Z","iopub.status.idle":"2022-11-04T18:14:06.970210Z","shell.execute_reply.started":"2022-11-04T18:14:06.956060Z","shell.execute_reply":"2022-11-04T18:14:06.969185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Code which goes through the results collected during training and plots them.\ndef plotResults(trainingResults):\n\n    metrics_available = trainingResults[list(trainingResults.keys())[0]].keys()\n\n    for i, metric in enumerate(metrics_available):\n        \n        plt.figure(figsize=(10,5))\n\n        plt.title(metric + \" over the course of training\")\n        plt.xlabel(\"Epoch\")\n        plt.ylabel(metric)\n        # Values were recorded for train and validation results\n        for phase in trainingResults.keys():\n\n            epochs = len(trainingResults[phase][metric])\n            epochs_range = range(1,epochs+1)\n\n            plt.plot(epochs_range,trainingResults[phase][metric],label=phase)\n\n        plt.legend()\n        plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.971871Z","iopub.execute_input":"2022-11-04T18:14:06.972248Z","iopub.status.idle":"2022-11-04T18:14:06.985181Z","shell.execute_reply.started":"2022-11-04T18:14:06.972210Z","shell.execute_reply":"2022-11-04T18:14:06.984062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def count_correct(preds, labels, debug=False):\n\n    correct_count = 0.0\n    \n    predicted_labels = torch.argmax(preds, dim=1)\n    correct_count += (predicted_labels == labels).sum().item()\n    \n    if debug:\n        print(\"Predicted:\",toNumpy(predicted_labels))\n        print(\"Labels:\",toNumpy(labels))\n        print(\"Correct Count:\",correct_count)\n    \n    return correct_count","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.986363Z","iopub.execute_input":"2022-11-04T18:14:06.988539Z","iopub.status.idle":"2022-11-04T18:14:06.995688Z","shell.execute_reply.started":"2022-11-04T18:14:06.988367Z","shell.execute_reply":"2022-11-04T18:14:06.994455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import albumentations as AUGS\nfrom sklearn import metrics\n\ndef train(model_class,model_init_params,Xtr,Ytr,valSplitSize=0.2,batch_size=32,num_epochs=20,lr=1e-3,verbose=0):\n    \n    X, Y = None, None\n    \n    trainSize = int((1.0 - valSplitSize)*len(Xtr))\n    print(\"Train set size:\",trainSize)\n    # Validation set\n    Xv, Yv = Xtr[trainSize:], Ytr[trainSize:]\n    # Train set\n    Xtr, Ytr = Xtr[:trainSize], Ytr[:trainSize]\n\n    print(\"Training Dataset sizes:\",\"[\", Xtr.shape, Ytr.shape,\"]\")\n    print(\"Validation Dataset sizes:\",\"[\", Xv.shape, Yv.shape,\"]\")\n    \n    device = torch.device(f'cuda')\n    \n    model = model_class(**model_init_params).to(device)\n    print(\"Model Loaded:\",model.name)\n    \n#     aug_transform = AUGS.Compose([\n#       # Vertical flipping\n#       # AUGS.VerticalFlip(),\n#       # AUGS.HueSaturationValue(p=0.2),       \n#       # AUGS.RandomBrightness(limit=0.2)\n#       AUGS.Affine(translate_percent=(0.0, 0.2), p=0.2),\n#       AUGS.Blur(blur_limit=3, always_apply=False, p=0.3)\n#       ])\n    \n    aug_transform=None\n    \n    # Make datasets\n    train_dataset = CustomDataset(Xtr, Ytr, num_classes=model.num_classes, aug_transform=aug_transform)\n    val_dataset = CustomDataset(Xv, Yv, num_classes=model.num_classes)\n    \n    # Make loaders\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, \n                              shuffle=True, num_workers=0) \n    \n    val_loader = DataLoader(val_dataset, batch_size=batch_size, \n                              shuffle=True, num_workers=0) \n    \n    criterion = nn.CrossEntropyLoss()\n    print(\"Loss Function:\",criterion)\n    \n        # Uses our learning rate parameter\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n    \n    results = dict()\n    for phase in [\"training\",\"validation\"]:\n        results[phase] = {\"loss\": [], \"accuracy\": []}\n    \n    lowestValidationAcc = np.NINF\n    \n    # Train the model\n    for epoch in range(num_epochs):\n        \n        results_this_epoch = dict()\n        for phase in [\"training\",\"validation\"]:\n            results_this_epoch[phase] = {\"loss\": [], \"accuracy\": []}\n        \n        # Needed since we switch to .eval() during the validation stage\n        model.train()\n        for i, (images, labels) in enumerate(tqdm(train_loader)):\n\n            images = images.to(device)\n            labels = labels.to(device)\n            # Forward pass\n            preds = model(images)\n            \n            loss = criterion(preds, labels)\n            # Backward and optimize\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            \n            results_this_epoch[\"training\"][\"loss\"].append(loss.item())\n            results_this_epoch[\"training\"][\"accuracy\"].append(count_correct(preds, labels))\n            \n        # Validation\n        \n        # Making sure we aren't using dropouts and also aren't trainined the model\n        model.eval()\n        with torch.no_grad():\n            \n            for i, (images, labels) in enumerate(tqdm(val_loader)):\n\n                images = images.to(device)\n                labels = labels.to(device)\n                # Forward pass\n                preds = model(images)\n\n                loss = criterion(preds, labels)\n                # No update step\n\n                results_this_epoch[\"validation\"][\"loss\"].append(loss.item())\n                results_this_epoch[\"validation\"][\"accuracy\"].append(count_correct(preds, labels))\n            \n        print('Epoch [{}/{}]'.format(epoch+1, num_epochs))\n\n        for phase in results_this_epoch.keys():\n            for metric in results_this_epoch[phase].keys():\n                if metric == \"loss\":\n                    results[phase][metric].append(np.mean(results_this_epoch[phase][metric]))\n                elif metric == \"accuracy\":\n                    if phase == \"training\":\n                        total = len(Ytr)\n                    elif phase == \"validation\":\n                        total = len(Yv)\n                    results[phase][metric].append(np.sum(results_this_epoch[phase][metric])/total)\n                    \n                if (verbose > 0):\n                    print('[{}] {}: {:.4f}'.format(phase, metric, results[phase][metric][-1]))\n                \n        # Early stopping, preventing overfitting when using too many epochs        \n        if results[\"validation\"][\"accuracy\"][-1] > lowestValidationAcc:\n            lowestValidationAcc = results[\"validation\"][\"accuracy\"][-1]\n            if verbose > 0:\n                print(\"  **\")\n            state = {'epoch': epoch + 1,\n                     'model_dict': model.state_dict(),\n                     'params': model_init_params,\n                     'best_loss_on_test': lowestValidationAcc}\n\n            modelLocation = model.name + \"_best_model.pth\"\n            torch.save(state, modelLocation) # saves to current directory\n            if verbose > 0:\n                  print(\"Model information (including weights) saved to:\",modelLocation)\n                    \n    plotResults(results)\n    \n    # Load up saved model\n    # model = model_class(**model_init_params).to(device)\n    # model.load_state_dict(torch.load(modelLocation,map_location=torch.device(device))[\"model_dict\"])\n    model.eval()\n    \n    return model\n","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:06.997274Z","iopub.execute_input":"2022-11-04T18:14:06.997992Z","iopub.status.idle":"2022-11-04T18:14:07.946085Z","shell.execute_reply.started":"2022-11-04T18:14:06.997945Z","shell.execute_reply":"2022-11-04T18:14:07.944995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trained_model = train(ConvNet,{},X,Y,verbose=1,num_epochs=15, batch_size=256, lr=1e-4, valSplitSize=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:14:07.949771Z","iopub.execute_input":"2022-11-04T18:14:07.950629Z","iopub.status.idle":"2022-11-04T18:15:44.388163Z","shell.execute_reply.started":"2022-11-04T18:14:07.950596Z","shell.execute_reply":"2022-11-04T18:15:44.387211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def infer_raw(filename, model):\n    \n    device = torch.device(f'cuda')\n\n    X, ids = load_data(filename, labels_present=False)\n\n    dataset = CustomDataset(X, np.zeros((len(X),)), num_classes=model.num_classes)\n\n    X = None # Free X\n\n    # Make loaders\n    data_loader = DataLoader(dataset, batch_size=128, shuffle=False, num_workers=0) \n\n    predicted_labels = np.asarray([],dtype=\"int\")\n    \n    model.eval()\n    with torch.no_grad():\n\n        for i, (images, _) in enumerate(tqdm(data_loader)):\n\n            images = images.to(device)\n            # Forward pass\n            preds = model(images)\n            \n            predicted_labels = np.concatenate( (predicted_labels, toNumpy(torch.argmax(preds, dim=1))) )            \n            \n    predicted_labels = predicted_labels.reshape(-1,1)\n    ids = ids.reshape(-1,1)\n    \n    return pd.DataFrame( np.concatenate( (ids, predicted_labels), axis=1 ), columns = [\"id\",\"label\"] )","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:15:44.389485Z","iopub.execute_input":"2022-11-04T18:15:44.390360Z","iopub.status.idle":"2022-11-04T18:15:44.399968Z","shell.execute_reply.started":"2022-11-04T18:15:44.390319Z","shell.execute_reply":"2022-11-04T18:15:44.398831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = infer_raw(data_dir + \"test.csv\", trained_model)\nresults.to_csv(\"./submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-04T18:15:44.401273Z","iopub.execute_input":"2022-11-04T18:15:44.401704Z","iopub.status.idle":"2022-11-04T18:15:44.978658Z","shell.execute_reply.started":"2022-11-04T18:15:44.401665Z","shell.execute_reply":"2022-11-04T18:15:44.977375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}