{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"markdown","source":"### Kaggle dataset setup","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"# Imports\nimport numpy as np \nimport pandas as pd\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\n\nimport torchvision\nfrom torchvision import transforms\n\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport seaborn as sn","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Config","metadata":{}},{"cell_type":"code","source":"### Config ###\n\n# Filepath Config\nDATAPATH=\"../input/Kannada-MNIST/\"\nTRAIN_FILE=\"train.csv\"\nTEST_FILE=\"test.csv\"\nSAMPLE_FILE =\"sample_submission.csv\"\nDIG_FILE=\"Dig-MNIST.csv\"\n\n# Data Config\nSPLITSIZE = 0.2\nNUM_CLASSES = 10\nCLASS_NAMES = {0:\"Omdu\", 1:\"Eradu\", 2:\"Muru\", 3:\"Nalku\", 4:\"Aidu\", 5:\"Aru\", 6:\"Elu\", 7:\"Emtu\", 8:\"Ombattu\", 9:\"Hattu\"}\nIMAGE_SIZE = 28\n\n# Model Config\nINITIAL_LR = 0.001\nBATCH_SIZE = 100\nEPOCHS = 50","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Acquire any available devices (i.e. GPU)","metadata":{}},{"cell_type":"code","source":"# Acquire any available GPUs\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(\"Current device type: \", device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{}},{"cell_type":"markdown","source":"### Data utility functions/classes","metadata":{}},{"cell_type":"code","source":"def load_dataset(file_name, datapath):\n    \"\"\"Read input CSV file for a given file name\"\"\"\n    data = pd.read_csv(datapath + file_name)\n    return data\n\n\ndef plot_image(label, classnames, image):\n    \"\"\"Plot an example image and class from the dataset, and save it to file\"\"\"\n    fig1 = plt.figure(figsize=(6,6));\n    fig1.tight_layout()\n    print(label)\n    plt.title(f\"Class: {label}, Name: {classnames[label]}\")\n    plt.imshow(image.to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')\n\n    \n    \nclass KannadaDataSet(torch.utils.data.Dataset):\n    \"\"\"Class for handling input CSVs as images using PIL; also handles specified image augmentation\"\"\"\n    # Get the passed image vector, label vector and transform config\n    def __init__(self, images, image_size, labels, transforms=None):\n        self.X = images\n        self.image_size = image_size\n        self.y = labels\n        self.transforms = transforms\n        \n    def __len__(self):\n        return len(self.X)\n    \n    def __getitem__(self, i):\n        # Get a single image vector\n        data = self.X.iloc[i, :] \n        # Reshape the vector into an image of shape 28*28\n        data = np.array(data).astype(np.uint8).reshape(self.image_size, self.image_size, 1) \n        \n        # Produce any transforms that are provided\n        if self.transforms:\n            data = self.transforms(data)\n        \n        # If the data is a training set, provide the label; otherwise do not\n        if self.y is not None: # train/val\n            return (data, self.y[i])\n        else:\n            return data","metadata":{"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model training utility functions","metadata":{}},{"cell_type":"code","source":"def train_model(network, device, optimizer, scheduler, training_data, batch_size):\n    \"\"\"Trains the model using the training data\"\"\"\n\n    epoch_loss = 0\n    epoch_correct = 0\n    network.train() # training mode\n    \n    for images, labels in training_data:\n        X, y = images.to(device), labels.to(device) # put X & y on device\n        y_ = network(X) # get predictions\n        \n        # Zeros the gradients\n        optimizer.zero_grad() \n        # Calculates the loss\n        loss = F.cross_entropy(y_, y) \n        # Computes the gradients\n        loss.backward() \n        # Update weights\n        optimizer.step() \n        \n        epoch_loss += loss.item() * batch_size\n        epoch_correct += get_num_correct(y_, y)   \n        \n    print(\"Train Loss: \", epoch_loss/BATCH_SIZE)\n    print(\"Train Acc:  \", epoch_correct/len(training_data))\n    \n    scheduler.step()\n    return optimizer.param_groups[0][\"lr\"], epoch_loss, epoch_correct/len(training_data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model evaluation utility functions","metadata":{}},{"cell_type":"code","source":"def get_num_correct(predictions, labels):\n    \"\"\"Compares model predictions with actual labels, returns the number of matches\"\"\"\n    return predictions.argmax(dim=1).eq(labels).sum().item()\n\n\ndef eval_model(network, device, validation_data, validation_set_size, batch_size):\n    \"\"\"Evaluation with the validation set\"\"\"\n    # Ensure the model is in eval mode (this disables dropout/batchnorm etc.)\n    network.eval() \n    val_loss = 0\n    val_correct = 0\n    \n    # Set all requires_grad() flags to false\n    with torch.no_grad():\n        # Loop through our validation data, generate predictions and \n        # add to the the loss/accuracy count for each image\n        for images, labels in validation_data:\n            X, y = images.to(device), labels.to(device) # to device\n            \n            # Get predictions\n            preds = network(X) \n            # Calculate Loss\n            loss = F.cross_entropy(preds, y)\n            \n            val_correct += get_num_correct(preds, y)\n            val_loss = loss.item() * batch_size\n\n    # Print the loss and accuracy for the validation set\n    print(\"Validation Loss: \", val_loss/BATCH_SIZE)\n    print(\"Validation Acc:  \", (val_correct/validation_set_size)*100)\n\n    # Return loss and accuracy values\n    return val_loss, ((val_correct/validation_set_size)*100)\n\n\ndef confusion_matrix(num_classes, validation_labels, predictions):\n    \"\"\"Generate and save a confusion matrix for the Neural Network\"\"\"\n    \n    # Generate the confusion matrix\n    cmt = torch.zeros(num_classes, num_classes, dtype=torch.int32)\n    for i in range(len(validation_labels)):\n        cmt[validation_labels[i], predictions[i]] += 1\n\n    # Detatch from CPU and convert to numpy array \n    cmt = cmt.cpu().detach().numpy()\n\n    df_cm = pd.DataFrame(cmt/np.sum(cmt) *10, index = [i for i in range(num_classes)],\n                        columns = [i for i in range(num_classes)])\n    \n    # Plot the dataframe using seaborn\n    plt.figure(figsize=(12,7))\n    sn.heatmap(df_cm, annot=True)\n    plt.title(\"Neural Network Confusion Matrix\", fontsize=20)\n\ndef get_n_params(model):\n    \"\"\"Summarizes the number of parameters in the neural network\"\"\"\n    \n    params_counter = 0\n    for param in list(model.parameters()):\n        n=1\n        for s in list(param.size()):\n            n = n*s\n        params_counter += n\n    return params_counter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Data","metadata":{}},{"cell_type":"markdown","source":"### Loading the csv data and correcting any index issues","metadata":{}},{"cell_type":"code","source":"train_set   = load_dataset(TRAIN_FILE, DATAPATH)\ntest_images = load_dataset(TEST_FILE, DATAPATH)\nval_set     = load_dataset(DIG_FILE, DATAPATH)\n\nprint(train_set.shape)\nprint(test_images.shape)\nprint(val_set.shape)\n\n# train_set   = pd.concat([train_set, val_set], axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the data into train/test splits - splitsize defined in config\ntrain_images, val_images, train_labels, val_labels = train_test_split(train_set.iloc[:, 1:], \n                                                                     train_set.iloc[:, 0], \n                                                                     test_size=SPLITSIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reset the index values\ntrain_images.reset_index(drop=True, inplace=True)\nval_images.reset_index(drop=True, inplace=True)\ntrain_labels.reset_index(drop=True, inplace=True)\nval_labels.reset_index(drop=True, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove ID column\ntest_images = test_images.iloc[:, 1:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot a sample image, along with label and class","metadata":{}},{"cell_type":"code","source":"# Plot a sample image, save it to the output folder\nplot_image(train_labels[0], CLASS_NAMES, train_images.iloc[[0], :])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Performing Image Normalization and Augmentations","metadata":{}},{"cell_type":"code","source":"# Define transform steps to convert csv data to images, and augment training data\n# ToTensor() Normalizes the image\ntrain_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.RandomCrop(IMAGE_SIZE),\n    transforms.RandomRotation(20),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1)),\n    transforms.ToTensor(),\n]))\n\n# No augmentations for the validation test set\n# ToTensor() Normalizes the image\nval_trans = transforms.Compose(([\n    transforms.ToPILImage(),\n    transforms.ToTensor(),\n]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Perform our transforms and augmentations\ntrain_set = KannadaDataSet(train_images, IMAGE_SIZE, train_labels, train_trans)\nval_set = KannadaDataSet(val_images, IMAGE_SIZE, val_labels, val_trans)\n\n# No labels for the test set\ntest_set = KannadaDataSet(test_images, IMAGE_SIZE, None, val_trans)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating the Neural Network","metadata":{}},{"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self):\n        super(Network, self).__init__()\n        # Image starts as a matrix of size (1, 28, 28)\n        \n        # The size of the image after each convolution or pooling layer can be obtained by: \n        # output = ((input - kernel_size + (2 * padding)) / stride) + 1\n        \n        # Convolutions and batch normalisations \n        # Batch norm reduces internal covariate shift\n        # Normalises the input feature (subtract batch mean, divide by batch sd)\n        # This speeds up neural network training times\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5, stride=1, padding=2) # conv1\n        self.conv1_bn = nn.BatchNorm2d(num_features=32)\n        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2) # conv2\n        self.conv2_bn = nn.BatchNorm2d(num_features=64)\n        self.conv3 = nn.Conv2d(in_channels=64, out_channels= 128, kernel_size=3, stride=1, padding=1) # conv3\n        self.conv3_bn = nn.BatchNorm2d(num_features=128)\n        self.conv4 = nn.Conv2d(in_channels=128, out_channels= 256, kernel_size=2, stride=1, padding=1) # conv4\n        self.conv4_bn = nn.BatchNorm2d(num_features=256)\n        \n        # Fully connected linear layers and batch normalisations\n        self.fc1 = nn.Linear(in_features=256*4*4, out_features=1024) # linear 1\n        self.fc1_bn = nn.BatchNorm1d(num_features=1024)\n        self.fc2 = nn.Linear(in_features=1024, out_features=512) # linear 2\n        self.fc2_bn = nn.BatchNorm1d(num_features=512)\n        self.fc3 = nn.Linear(in_features=512, out_features=256) # linear 3\n        self.fc3_bn = nn.BatchNorm1d(num_features=256)\n        self.fc4 = nn.Linear(in_features=256, out_features=64) # linear 4\n        self.fc4_bn = nn.BatchNorm1d(num_features=64)\n        \n        # Final Layer\n        self.out = nn.Linear(in_features=64, out_features=10) # output\n        \n        # Dropout\n        self.dropout = nn.Dropout(0.4)\n    \n    def forward(self, z):\n        \n        # Apply Relu then Max Pool function between each convolution layer\n        z = F.relu(self.conv1_bn(self.conv1(z)))            # (1, 28, 28)\n        z = F.max_pool2d(z, kernel_size=2, stride=2)        # (1, 14, 14)\n        \n        z = F.relu(self.conv2_bn(self.conv2(z)))            # (1, 14, 14)\n        z = F.max_pool2d(z, kernel_size=2, stride=2)        # (1, 7, 7)\n        \n        z = F.relu(self.conv3_bn(self.conv3(z)))            # (1, 7, 7)\n        z = F.max_pool2d(z, kernel_size=2, stride=1)        # (1, 6, 6)\n        \n        z = F.relu(self.conv4_bn(self.conv4(z)))            # (1, 7, 7)\n        z = F.max_pool2d(z, kernel_size=4, stride=1)        # (1, 4, 4)\n        \n        # Apply Relu function between each fully connected layer\n        #print(z.size())\n        z = F.relu(self.fc1_bn(self.fc1(z.reshape(-1, 256*4*4)))) # 256 4 4\n        z = self.dropout(z)\n        \n        z = F.relu(self.fc2_bn(self.fc2(z)))\n        z = self.dropout(z)\n        \n        z = F.relu(self.fc3_bn(self.fc3(z)))\n        z = self.dropout(z)\n        \n        z = F.relu(self.fc4_bn(self.fc4(z)))\n        z = self.out(z)\n        \n        return z","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ","metadata":{}},{"cell_type":"markdown","source":"# Model Training","metadata":{}},{"cell_type":"markdown","source":"### Create the model on the available device","metadata":{}},{"cell_type":"code","source":"net = Network().to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get a summary of the number of parameters in the model","metadata":{}},{"cell_type":"code","source":"params = get_n_params(net)\n\nprint(f\"There are {params} total parameters in the neural network.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Generate the training/validation dataloaders","metadata":{}},{"cell_type":"code","source":"# Create dataset dataloaders\ntraining_dataloader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)\nvalidation_dataloader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define the model optimiser - we're using Adam here","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(net.parameters(), lr=INITIAL_LR)\n# optimizer = optim.Adagrad(net.parameters(), lr=INITIAL_LR)\n# optimizer = optim.SGD(net.parameters(), lr=INITIAL_LR, nesterov=True, momentum=0.9)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define the Learning Rate Scheduler","metadata":{}},{"cell_type":"code","source":"scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\n# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train the model","metadata":{}},{"cell_type":"code","source":"lrs = []\ntrain_loss = []\ntrain_acc = []\nval_loss = []\nval_acc = []\n\n# Run our epoch cycles\nfor epoch in range(EPOCHS):\n    # Print epoch cycle\n    print(\"===============================================\")\n    print(f\"Epoch Cycle: {epoch+1}\")\n    print(\"-----------------------------------------------\")\n    \n    # Train the model, and append the current learning rate\n    lr, tl, ta = train_model(net, device, optimizer, scheduler, training_dataloader, BATCH_SIZE)\n    \n    lrs.append(lr)\n    train_loss.append(tl/BATCH_SIZE)\n    train_acc.append(ta)\n    \n    # Evaluate the model, return the validation loss and validation accuracy\n    vl, va = eval_model(net, device, validation_dataloader, len(val_images), BATCH_SIZE)\n    \n    val_loss.append(vl/BATCH_SIZE)\n    val_acc.append(va)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation","metadata":{}},{"cell_type":"code","source":"# Create empty tensor for predictions\npredictions = torch.LongTensor().to(device)\n\n# Use trained model to generate predictions\nfor images, _ in validation_dataloader:\n    preds = net(images.to(device))\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### View the model's confusion matrix","metadata":{}},{"cell_type":"code","source":"# Plot confusion matrix\nconfusion_matrix(NUM_CLASSES, val_labels, predictions)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create list of epochs for plotting purposes\nepoch_num = list(range(1, EPOCHS+1))\n\nplt.figure(figsize=(12,7))\nplt.title(\"Neural Network Learning Rate Curve\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Learning Rate Value\")\nplt.grid()\nplt.plot(epoch_num, lrs, 'g-')\nplt.xticks(epoch_num)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Plot the model's training/loss curves","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12,7))\nplt.title(\"Neural Network Loss Curves\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss Value\")\nplt.grid()\nplt.plot(epoch_num, train_loss, 'b-', label='training loss')\nplt.plot(epoch_num, val_loss, 'r-', label='validation loss')\nplt.legend(loc=\"upper right\")\nplt.xticks(epoch_num)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,7))\nplt.title(\"Neural Network Accuracy Curves\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy (%)\")\nplt.ylim(97,100)\nplt.grid()\nplt.plot(epoch_num, train_acc, 'b-', label='training accuracy')\nplt.plot(epoch_num, val_acc, 'r-', label='validation accuracy')\nplt.legend(loc=\"lower right\")\nplt.xticks(epoch_num)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate submission file","metadata":{}},{"cell_type":"code","source":"# Add the test set to a DataLoader\ntest_dl = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False)\n\n# Make sure the model is in eval mode!\nnet.eval()\n\n# Create a tensor for predictions\npredictions = torch.LongTensor().to(device) \n\n# Save the new predictions\nfor images in test_dl:\n    preds = net(images.to(device))\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Edit the existing sample submission file with our own predictions\nsubmission_file = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Rewrite the label column with our predictions\n# As predictions is a tensor on the CPU, covert it to a numpy array to be saved\nsubmission_file['label'] = predictions.cpu().numpy()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the dataframe as a new submission csv!\nsubmission_file.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}