{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## *Handwritten digit recognition using Deep Learning Frameworks*","metadata":{}},{"cell_type":"markdown","source":"In this Notebook we try to identify digits from a dataset of thousands of handwritten images. To this end, we develop two classic deep learning models, a *Multi Layer Perceptron (MLP)* and a *Convolution Neural Network (CNN)*. We develop several dataloading pipelines and experiment with various techniques such as data augmentation and different training procedures. At the end we achieve a score more than 99 % at the test set. Finally, we make predictions to unlabeled data and visualize the results.","metadata":{}},{"cell_type":"markdown","source":"## Part 1. Data Loading & Visualizations","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport matplotlib.pyplot as plt\nimport os\n\npath_train = os.path.join(\"../input/digit-recognizer\",\"train.csv\")\npath_test = os.path.join(\"../input/digit-recognizer\",\"test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.loadtxt(path_train,skiprows=1, delimiter=\",\")\nprint(f\"- The dataset shape is equal to {data.shape}\")\nX,y = data[:,1:]/255, data[:,0].astype(\"int64\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the unique digits\nnp.unique(y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"indices = []\nfor digit in np.unique(y):\n    indices.append(np.where(y == digit)[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualizing some samples\nfig, ax = plt.subplots(figsize = (12,10), nrows = 2, ncols =5)\nfor i,idx in enumerate(indices):\n    r,c = divmod(i,5)\n    img = X[idx,:]\n    img.resize((28,28))\n    ax[r,c].imshow(img, cmap = \"gray\")\n    ax[r,c].axis(\"off\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 2. Deep Learning models","metadata":{}},{"cell_type":"markdown","source":"Preparing the training, validation and test sets","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(\n    X,y, test_size = 0.3, random_state=42\n)\nX_test, X_val, y_test, y_val = train_test_split(\n    X_val, y_val, test_size = 0.5, random_state=42\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training size: {X_train.shape[0]}, Validation size: {X_val.shape[0]}, Test size: {X_test.shape[0]}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.optim import Adam\nimport seaborn as sns\nfrom torch.optim.lr_scheduler import StepLR","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The training loop\n\ndef training_loop(model, train_loader, val_loader, epochs,\n                  lr, loss_fn, regularization=None,\n                  reg_lambda=None, mod_epochs=20, early_stopping = False,\n                  patience = None, verbose = None, title = None,\n                  scheduler_bool = False, gamma = 0.1, step_size = 10, model_name = \"model\"):\n    optim = Adam(model.parameters(), lr=lr)\n    if scheduler_bool:\n        scheduler = StepLR(optim, step_size = step_size, gamma = gamma)\n    if torch.cuda.is_available():\n        device = \"cuda\"\n    else:\n        device = \"cpu\"\n\n    train_loss_list = []\n    val_loss_list = []\n    num_train_batches = len(train_loader)\n    num_val_batches = len(val_loader)\n    counter_epochs = 0\n\n    if early_stopping:\n        ear_stopping = EarlyStopping(patience= patience, verbose=verbose, path = model_name+\".pt\")\n\n    for epoch in range(epochs):\n        counter_epochs+=1\n        model.train()\n        train_loss, val_loss = 0.0, 0.0\n        for train_batch in train_loader:\n            X, y = train_batch[0].to(device), train_batch[1].to(device)\n            preds = model(X)\n            loss = loss_fn(preds, y)\n            train_loss += loss.item()\n\n            # Regulirization\n            if regularization == 'L2':\n                l_norm = sum(p.pow(2.0).sum() for p in model.parameters())\n                loss = loss + reg_lambda * l_norm\n            elif regularization == 'L1':\n                l_norm = sum(p.abs().sum() for p in model.parameters())\n                loss = loss + reg_lambda * l_norm\n\n            # Backpropagation\n            optim.zero_grad()\n            loss.backward()\n            optim.step()\n        model.eval()\n        with torch.no_grad():\n            for val_batch in val_loader:\n                X, y = val_batch[0].to(device), val_batch[1].to(device)\n                preds = model(X)\n                val_loss += loss_fn(preds, y).item()\n        train_loss /= num_train_batches\n        val_loss /= num_val_batches\n        train_loss_list.append(train_loss)\n        val_loss_list.append(val_loss)\n        if (epoch + 1) % mod_epochs == 0:\n            print(\n                f\"Epoch: {epoch + 1}/{epochs}{5 * ' '}Training Loss: {train_loss:.4f}{5 * ' '}Validation Loss: {val_loss:.4f}\")\n\n        if early_stopping:\n            ear_stopping(val_loss, model)\n            if ear_stopping.early_stop:\n                print(\"Early stopping\")\n                break\n        if scheduler_bool:\n            scheduler.step()\n    sns.set_style(\"dark\")\n    fig, ax = plt.subplots(figsize=(8, 6))\n    ax.plot(range(1, counter_epochs + 1), train_loss_list, label='Train Loss')\n    ax.plot(range(1, counter_epochs + 1), val_loss_list, label='Val Loss')\n    ax.set_title(\"Train - Val Loss\")\n    ax.set_ylabel(\"Loss\")\n    ax.set_xlabel(\"Epochs\")\n    plt.legend()\n    plt.show()\n\n    if early_stopping:\n        model.load_state_dict(torch.load(model_name+\".pt\"))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Early stopping\n\n# Source: https://github.com/Bjarten/early-stopping-pytorch/blob/master/pytorchtools.py\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, patience=7, verbose=False, delta=0, path='checkpoint.pt', trace_func=print):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement.\n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n            path (str): Path for the checkpoint to be saved to.\n                            Default: 'checkpoint.pt'\n            trace_func (function): trace print function.\n                            Default: print\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n        self.path = path\n        self.trace_func = trace_func\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            if self.counter % 5 == 0: # Print only per 5 degradations of val loss\n                self.trace_func(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            self.trace_func(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), self.path)\n        self.val_loss_min = val_loss","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The test loop \ndef test_loop(model, test_dloader, device='cpu'):\n    predictions_list = np.array([], dtype=np.int64)\n    targets_list = np.array([], dtype=np.int64)\n    model.eval()\n\n    for val_sample in test_dloader:\n        X = val_sample[0].to(device)\n        y = val_sample[1].cpu().numpy()\n        targets_list = np.concatenate((targets_list, y))\n\n        with torch.no_grad():\n            preds = model(X)\n            predictions_list = np.concatenate((predictions_list,\n                                               torch.argmax(preds, dim=-1).cpu().numpy()))\n    acc = len(targets_list[targets_list == predictions_list])/len(targets_list)\n    return predictions_list, targets_list, acc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.1. A Simple MLP","metadata":{}},{"cell_type":"code","source":"accs = [] # A list to hold the accuracy for the three models","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The Architecture of the MLP\nfrom typing import OrderedDict\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset\n\nclass MLP(nn.Module):\n    def __init__(self, in_features = 28*28, out_features = 10,\n                 hidden_layers = [512,256,128]):\n        super(MLP,self).__init__()\n        self.structure = OrderedDict()\n        for i,layer in enumerate(hidden_layers):\n            if i == 0:\n                self.structure[\"Linear \" + str(i+1)] = nn.Linear(in_features=in_features,\n                                                                    out_features=layer, bias=False)\n                self.structure[\"BatchNorm1D \" + str(i+1)] = nn.BatchNorm1d(layer)\n                self.structure[\"Relu \" + str(i+1)] = nn.ReLU()\n                self.structure[\"Dropout \" + str(i+1)] = nn.Dropout(p = 0.2)\n            else:\n                self.structure[\"Linear \" + str(i+1)] = nn.Linear(in_features = hidden_layers[i-1],\n                                                    out_features=layer, bias=False)\n                self.structure[\"BatchNorm1D \" + str(i+1)] = nn.BatchNorm1d(layer)\n                self.structure[\"Relu \" + str(i+1)] = nn.ReLU()\n                self.structure[\"Dropout \" + str(i+1)] = nn.Dropout(p = 0.2)\n        self.structure[\"Out Linear\"] = nn.Linear(in_features=hidden_layers[-1],out_features=out_features)\n        \n        self.linear_relu_stack = nn.Sequential(self.structure)\n        \n    def forward(self,x):\n        return self.linear_relu_stack(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dataset for the MLP\nclass mlp_dataset(Dataset):\n    def __init__(self, X, y):\n        self.X = X; self.y = y\n    \n    def __len__(self):\n        return len(self.y)\n    \n    def __getitem__(self,idx):\n        return torch.tensor(self.X[idx,:], dtype=torch.float32), torch.tensor(self.y[idx], dtype=torch.int64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Setting training Hyperparameters and preparing the datasets & dataloaders\nfrom torch.utils.data import DataLoader\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nBATCH_SIZE = 128\nLR = 1e-3\nPATIENCE = 20\nloss_fn = nn.CrossEntropyLoss().to(device)\nEPOCHS = 300\n\ntrain_dloader = DataLoader(mlp_dataset(X_train,y_train),\n                           batch_size=BATCH_SIZE, shuffle=True)\ntest_dloader = DataLoader(mlp_dataset(X_test,y_test),batch_size=BATCH_SIZE)\nval_dloader = DataLoader(mlp_dataset(X_val, y_val), batch_size=BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MLP(hidden_layers=[512,256,128,64]).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Start training!!\ntraining_loop(model, train_dloader, val_dloader, epochs=EPOCHS, lr = LR,\n              loss_fn=loss_fn, regularization=\"L2\", reg_lambda=1e-4, mod_epochs=10,\n              early_stopping=True, verbose=True, patience=PATIENCE, title = \"MLP training\",\n              scheduler_bool=True, model_name = \"model1\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate on the test set (not the submission test)\nfrom sklearn.metrics import classification_report\n\npreds, targets, acc = test_loop(model,test_dloader,device)\naccs.append(acc)\nprint(classification_report(targets, preds))\nprint(f\"Accuracy score: {100*acc:.2f} %\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2. CNN ","metadata":{}},{"cell_type":"code","source":"# The main architecture\n\nclass conv_block(nn.Module):\n    def __init__(self, in_channels, out_channels,\n                 kernel_size = 5, stride = 1, padding = 0, bias = True):\n        super(conv_block,self).__init__()\n        \n        self.conv = nn.Sequential(\n            nn.Conv2d(in_channels = in_channels, out_channels =  out_channels,\n                      kernel_size = kernel_size, stride = stride,\n                      padding = padding, bias = bias),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU()\n            )\n    def forward(self,x):\n        return self.conv(x)\n        \n        \nclass CNN(nn.Module):\n    def __init__(self,in_channels, out_channels):\n        super(CNN,self).__init__()\n        self.backbone = nn.Sequential(\n            conv_block(in_channels=in_channels,\n                       out_channels=6, bias = False),\n            nn.AdaptiveMaxPool2d(output_size = 12),\n            conv_block(in_channels = 6, out_channels = 16,\n                       bias = False),\n            nn.AdaptiveMaxPool2d(output_size = 4),\n            nn.Flatten()\n        )\n        self.classifier = nn.Sequential(\n            nn.Linear(in_features=4*4*16, out_features=256, bias=False),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Linear(in_features = 256, out_features = 128, bias = False),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Linear(in_features = 128, out_features = 10)\n        )\n    \n    def forward(self,x):\n        x = self.backbone(x)\n        return self.classifier(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The dataset for the CNN\n\nclass cnn_dataset(Dataset):\n    def __init__(self,X,y, transforms = None):\n        self.X = np.expand_dims(X, axis = 0); self.y = y\n        self.transforms = transforms\n    def __len__(self):\n        return len(self.y)\n    \n    def __getitem__(self,idx):\n        if self.transforms is not None:\n            img = np.resize(self.X[:,idx,:], (28,28,1))\n            augmentations = self.transforms\n            features = augmentations(image = img)[\"image\"]\n            features = torch.tensor(np.transpose(features, (2,0,1)), dtype=torch.float32)\n            labels = torch.tensor(self.y[idx], dtype = torch.int64)\n            return features, labels\n        else:\n            features = torch.tensor(self.X[:,idx,:], dtype = torch.float32).view(-1,28,28)\n            label = torch.tensor(self.y[idx], dtype = torch.int64)\n            return features, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preparing datasets & dataloaders\ntrain_dloader = DataLoader(cnn_dataset(X_train,y_train),\n                           batch_size=BATCH_SIZE, shuffle=True)\ntest_dloader = DataLoader(cnn_dataset(X_test,y_test),batch_size=BATCH_SIZE)\nval_dloader = DataLoader(cnn_dataset(X_val, y_val), batch_size=BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN(in_channels=1, out_channels=10).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_loop(model, train_dloader, val_dloader, epochs=EPOCHS, lr = LR,\n              loss_fn=loss_fn, regularization=\"L2\", reg_lambda=1e-4, mod_epochs=10,\n              early_stopping=True, verbose=True, patience=PATIENCE, title = \"CNN Training\",\n              scheduler_bool=True, model_name=\"model2\", step_size = 20)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\npreds, targets, acc = test_loop(model,test_dloader,device)\naccs.append(acc)\nprint(classification_report(targets, preds))\nprint(f\"Accuracy score: {100*acc:.2f} %\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3. Applying Data Augmentation","metadata":{}},{"cell_type":"code","source":"import albumentations as A","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = [A.HorizontalFlip(p=1),\n              A.OpticalDistortion(distort_limit=1.2,shift_limit=0.6,p=1),\n              A.PixelDropout(dropout_prob=0.01,p=1)]\n\ntransforms_names = [\"Horizontal Flip\", \"Optical Distortion\",\n                    \"PixelDropout\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\nimport seaborn as sns\n\ndef visualize_tranforms(digit, X, y, transforms, random_state = 42):\n    indices = np.where(y == digit)[0]\n    random.seed(random_state)\n    idx = indices[random.randint(a=0,b=len(indices)-1)]\n    img = np.resize(X[idx,:], (28,28,1))\n    sns.set_style(\"dark\")\n    fig, ax = plt.subplots(figsize = (10,5), nrows = 1, ncols = len(transforms)+1)\n    for i in range(len(transforms)+1):\n        if i==0:\n            ax[i].imshow(img, cmap = \"gray\")\n            ax[i].axis(\"off\")\n            ax[i].set_title(\"original\")\n        else:\n            transformed_img = transforms[i-1](image = img)[\"image\"]\n            ax[i].imshow(transformed_img, cmap = \"gray\")\n            ax[i].axis(\"off\")\n            ax[i].set_title(transforms_names[i-1])\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tranforms(digit = 6,X = X,y = y,transforms=transforms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tranforms(digit = 9,X = X,y = y,transforms=transforms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_tranforms(digit = 0,X = X,y = y,transforms=transforms)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = A.Compose([\n    A.OpticalDistortion(distort_limit=1.2,shift_limit=0.6,p=0.5),\n    A.OneOf([A.HorizontalFlip(p=1),\n             A.PixelDropout(dropout_prob=0.01,p=1)\n             ], p = 0.5)\n])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dloader = DataLoader(cnn_dataset(X_train,y_train, transforms=transforms),\n                           batch_size=BATCH_SIZE, shuffle=True)\ntest_dloader = DataLoader(cnn_dataset(X_test,y_test),batch_size=BATCH_SIZE)\nval_dloader = DataLoader(cnn_dataset(X_val, y_val), batch_size=BATCH_SIZE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN(in_channels=1, out_channels=10).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_loop(model, train_dloader, val_dloader, epochs=EPOCHS, lr = LR,\n              loss_fn=loss_fn, regularization=\"L2\", reg_lambda=1e-4, mod_epochs=10,\n              early_stopping=True, verbose=False, patience=PATIENCE, title = \"CNN Training with augmentations\",\n              scheduler_bool=True, step_size = 20, model_name = \"model3\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\n\npreds, targets, acc = test_loop(model,test_dloader,device)\naccs.append(acc)\nprint(classification_report(targets, preds))\nprint(f\"Accuracy score: {100*acc:.2f} %\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3. Inference & Submissions","metadata":{}},{"cell_type":"markdown","source":"### 3.1. Inference","metadata":{}},{"cell_type":"code","source":"# Finding the best model\nmax_acc = max(accs)\nidx = accs.index(max_acc)\nprint(f\"- The highest accuracy is: {100*max_acc:.2f} by model{idx+1}\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = np.loadtxt(path_test, skiprows=1, delimiter = \",\")\nprint(f\"Test contains {test.shape[0]} samples\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load best model - CNN without data augmentation\nif idx ==0:\n    best_model = MLP(hidden_layers=[512,256,128,64]).to(device)\n    best_model.load_state_dict(torch.load(\"model\"+str(idx+1)+\".pt\"))\nelse:\n    best_model = CNN(in_channels=1, out_channels=10).to(device)\n    best_model.load_state_dict(torch.load(\"model\"+str(idx+1)+\".pt\"))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class submission_dataset(nn.Module):\n    def __init__(self,test_set):\n        self.test_set = test_set\n    def __len__(self):\n        return self.test_set.shape[0]\n    def __getitem__(self,idx):\n        img = torch.tensor(np.expand_dims(self.test_set[idx,:], axis = 0), dtype=torch.float32)\n        return img.view(-1,28,28)/255","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndef submission_loop(model,test, file = \"submission.csv\"):\n    sub_loader = DataLoader(submission_dataset(test), batch_size = 1)\n    device = \"cuda\" if torch.cuda.is_available else \"cpu\"\n    model.eval()\n    submissions = np.zeros(shape = (test.shape[0], 2), dtype=np.int64)\n    with torch.no_grad():\n        for sample_id, img in enumerate(sub_loader):\n            img = img.to(device)\n            pred_digit = int(torch.argmax(model(img), dim = -1).cpu().numpy())\n            submissions[sample_id, 0] = sample_id +1\n            submissions[sample_id, 1] = pred_digit\n    df_submissions = pd.DataFrame({\"ImageId\": submissions[:,0],\n                                   \"Label\": submissions[:,1]})\n    df_submissions.to_csv(file, sep = \",\", header=True, index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_loop(best_model, test)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2. Visualizing Predictions","metadata":{}},{"cell_type":"code","source":"submissions = np.loadtxt(\"submission.csv\", skiprows=1,\n                         dtype=np.int64, delimiter=\",\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def visualize_preds(test, submissions, factor = 5,\n                    seed = 24, ncols = 4):\n    fig,ax = plt.subplots(figsize = (10,10), nrows = factor,\n                      ncols = ncols)\n    np.random.seed(seed)\n    indices = np.random.randint(low = 0, high = test.shape[0],\n                                size = factor*ncols)\n    for i, idx in enumerate(indices):\n        r,c = divmod(i,ncols)\n        img = np.expand_dims(test[idx,:],axis = 0)\n        ax[r,c].imshow(np.resize(img,(28,28,1)),\n                    cmap = \"gray\")\n        ax[r,c].axis(\"off\")\n        ax[r,c].set_title(f\"ID:{idx +1}, Pred: {submissions[idx,1]}\",\n                        fontsize = 10)\n    plt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_preds(test, submissions, factor = 5,\n                seed = 42, ncols = 5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualize_preds(test, submissions, factor = 5,\n                seed = 23, ncols = 5)","metadata":{},"execution_count":null,"outputs":[]}]}