{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"In this notebook, we will use fastai library to create and train a Convolutional Neural Network.fastai is a deep learning library which provides practitioners with high-level components that can quickly and easily provide state-of-the-art results in standard deep learning domains\n\nDocumentation:\nhttps://docs.fast.ai/\n\nYou may notice that the code is quite similar to the notebook provided in: https://github.com/fastai/fastai/blob/master/dev_nbs/course/lesson5-sgd-mnist.ipynb. I adapted this code but try to add more detailed explanations to the steps that I am doing, to make it more beginner friendly"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%bash\n\n## Install fastai v2\n\npip install fastai2\npip install torchsummary\npip uninstall fastcore -y\npip install fastcore==1.0.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Import Necessary Libraries:"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport torch\nfrom fastai2.basics import *\nfrom fastai2.data.all import *\nfrom fastai2.callback.all import *\nfrom fastai2.vision.all import *\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nprint('Shape of the training data: ', train.shape)\nprint('Shape of the test data: ', test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's look at a sample digit"},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_categorical(y, num_classes):\n    \"\"\" 1-hot encodes a tensor \"\"\"\n    return np.eye(num_classes, dtype='uint8')[y]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train.drop(columns=['label']).values\nY_train = train['label'].values\nX_train = X_train / 255.0\nX_train = np.reshape(X_train, (X_train.shape[0], 28, 28))\nX_train = np.expand_dims(X_train, axis=1)\nX_test = test.values\nX_test = X_test  / 255.0\nX_test = np.reshape(X_test, (X_test.shape[0], 28, 28))\nX_test = np.expand_dims(X_test, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = train.groupby('label').apply(lambda x: x.sample(n=1)).reset_index(drop = True)\nsample_df.drop(columns=['label'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nrows = 2\nncols = 5\nfig, axs = plt.subplots(nrows=nrows, ncols=ncols, gridspec_kw={'wspace': 0.01, 'hspace': 0.05},\n                       squeeze=True, figsize=(10,12))\n\nind_y = 0\nind_x = 0\nfor i, row in sample_df.iterrows():\n    if ind_y > ncols - 1:\n        ind_y = 0\n        ind_x += 1\n    sample_digit = sample_df.values[i, :].reshape((28, 28))\n    axs[ind_x, ind_y].axis('off')\n    axs[ind_x, ind_y].imshow(sample_digit, cmap='gray')\n    axs[ind_x, ind_y].set_title(\"Digit {}:\".format(i))\n    ind_y += 1\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's analyze the distributions of digits in our dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(10,6))\nax = fig.add_subplot(111)\nax = sns.countplot(x=\"label\", data=train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looks like there is no severe imbalance of data in our dataset"},{"metadata":{},"cell_type":"markdown","source":"## Define Model"},{"metadata":{},"cell_type":"markdown","source":"I won't dive too much into details on choosing a CNN architecture as Chris' kernel already provides an excellent explanation. (See it here: https://www.kaggle.com/cdeotte/how-to-choose-cnn-architecture-mnist). I will go ahead and show how to define a model using fastai"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Derived from Chris' kernel, rewritten using pytorch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nBATCH_SIZE = 128\n\nclass Mnist_NN(nn.Module):\n    def __init__(self):\n        super(Mnist_NN, self).__init__()\n        self.conv2d_1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=3,\n                                 stride=1)\n        self.bn = nn.BatchNorm2d(32)\n        self.conv2d_2 = nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3)\n        self.relu = nn.ReLU()\n        self.bn_2 = nn.BatchNorm2d(32)\n        self.conv2d_3 = nn.Conv2d(in_channels=32, out_channels=32, kernel_size=5, stride=2)\n        self.bn_3 = nn.BatchNorm2d(32)\n        self.dropout = nn.Dropout(0.4)\n        self.conv2d_4 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3)\n        self.bn_4 = nn.BatchNorm2d(64)\n        self.conv2d_5 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3)\n        self.bn_5 = nn.BatchNorm2d(64)\n        self.conv2d_6 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=5, stride=2)\n        self.bn_6 = nn.BatchNorm2d(64)\n        # Fully connected 1\n        self.fc1 = nn.Linear(64*15*15, 128) \n        self.bn_7 = nn.BatchNorm1d(128)\n        self.fc2 = nn.Linear(128, 10) \n        \n\n    def forward(self, xb):\n        x = self.conv2d_1(xb)\n        x = self.relu(x)\n        x = self.bn(x)\n        x = self.conv2d_2(x)\n        x = self.relu(x)\n        x = self.bn_2(x)\n        x = self.conv2d_3(x)\n        x = self.relu(x)\n        x = F.pad(x, (0, 28, 28, 0))  # [left, right, top, bot]\n        x = self.bn_3(x)\n        x = nn.Dropout(p=0.4)(x)\n        x = self.conv2d_4(x)\n        x = self.relu(x)\n        x = self.bn_4(x)\n        x = self.conv2d_5(x)\n        x = self.relu(x)\n        x = self.bn_5(x)\n        x = self.conv2d_6(x)\n        x = self.relu(x)\n        x = self.bn_6(x)\n        x = self.relu(x)\n        x = nn.Dropout(p=0.4)(x)\n        # flatten\n        x = x.view(-1, x.size(1)*x.size(2)*x.size(3))\n        x = self.fc1(x)\n        x = self.relu(x)\n        x = nn.Dropout(p=0.4)(x)\n        x = self.fc2(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check if model parameters have been set correctly\n\n# from torchsummary import summary\n\n\n# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n# model = Mnist_NN().to(device)\n# summary(model, input_size=(1, 28, 28))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Perform train, validation split\nx_train, x_valid, y_train, y_valid = train_test_split(X_train, Y_train, test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = torch.from_numpy(x_train).float()\ny_train = torch.from_numpy(y_train)\ny_train = y_train.squeeze()\nx_valid = torch.from_numpy(x_valid).float()\ny_valid = torch.from_numpy(y_valid) \ny_valid = y_valid.squeeze()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Define our datasets and dataloaders below.\n\n**Dataset** - Dataset wrapping tensors. Each sample will be retrieved by indexing tensors along the first dimension.\n\n**TfmdDL** - A TfmdDL is a DataLoader that creates Pipeline from a list of Transforms for the callbacks after_item, before_batch and after_batch. As a result, it can decode or show a processed batch.\n\n**Dataloaders** - Basic wrapper around several DataLoader"},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import TensorDataset\n\ntrain_ds = TensorDataset(x_train, y_train)\nvalid_ds = TensorDataset(x_valid, y_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = TfmdDL(train_ds, bs=BATCH_SIZE, shuffle=True)\nvalid_dl = TfmdDL(valid_ds, bs=2*BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if torch.cuda.is_available():\n    dls = DataLoaders(train_dl, valid_dl).cuda()\nelse:\n    dls = DataLoaders(train_dl, valid_dl)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now, we define our fastai Learner. \n\nWhat is a fastai learner?\n* It is basically a training loop wrapper in a form of a class. It encapsulates training details / logs that helps in understanding metrics/loss output in each iteration"},{"metadata":{},"cell_type":"markdown","source":"We will use nn.CrossEntropyLoss as our loss function. This criterion combines nn.LogSoftmax() and nn.NLLLoss() in one single class."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define our loss function\n\nloss_func = nn.CrossEntropyLoss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nmodel = Mnist_NN().float()\nmodel.to(device)\nlearn = Learner(dls, model, loss_func=loss_func, metrics=accuracy)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For fastai, we can do something known as 1 cycle learning. In the paper, “A disciplined approach to neural network hyper-parameters: Part 1 — learning rate, batch size, momentum, and weight decay”, Leslie Smith suggests 1 Cycle Policy to apply learning rates. The idea is to tune learning rate, to go from a lower learning rate to a higher learning rate and back to a lower learning rate. The motivation behind this is that, during the middle of learning when learning rate is higher, the learning rate works as regularisation method and keep network from overfitting. This helps the network to avoid steep areas of loss and land better flatter minima."},{"metadata":{"trusted":true},"cell_type":"code","source":"from fastai2.callback.all import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Find optimal learning rate\nlearn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train learner \n\nlearn.unfreeze()\nlearn.model_dir = \"/kaggle/working\"\n\n# if GPU enabled:\nif torch.cuda.is_available():\n    learn.model.cuda()\n\nlearn.fit_one_cycle(5, 5e-2, callbacks=[SaveModelCallback(every='improvement', \n                                        monitor='validation_loss', name='best_saved')])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That is really powerful stuff! In just 5 epochs, we reach 0.967x validation accuracy. If we wish, we can train for more cycles.\n\nWe can call learn.lr_find() and train for more cycles to improve our model"},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.lr_find()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learn.fit_one_cycle(20, 5e-5, callbacks=[SaveModelCallback(every='improvement', \n                                        monitor='validation_loss', name='best_saved')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(2,1,figsize=(8,12))\nax[0].plot(list(range(95)),learn.sched.val_losses, label='Validation loss')\nax[0].plot(list(range(95)),[learn.sched.losses[i] for i in range(97,95*98,98)], label='Training \nloss')\nax[0].set_xlabel('Epoch')\nax[0].set_ylabel('Loss')\nax[0].legend(loc='upper right')\nax[1].plot(list(range(95)),learn.sched.rec_metrics)\nax[1].set_xlabel('Epoch')\nax[1].set_ylabel('Accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = TensorDataset(x_valid, y_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# From https://www.kaggle.com/puneetgrover/training-your-own-cnn-using-pytorch\ndef to_np(v):\n    \"\"\"\n    Converts Variable or Tensor to numpy array.\n    -------------------------------------------------------\n    Parameters:\n        v: A pyTorch Variable or Tensor\n    Output:\n        Return a numpy array\n    \"\"\"\n    if isinstance(v, (np.ndarray, np.generic)): return v\n    if isinstance(v, (list,tuple)): return [to_np(o) for o in v]\n    if isinstance(v, Variable): v=v.data\n    if torch.cuda.is_available():\n        if isinstance(v, torch.cuda.HalfTensor): v=v.float()\n    if isinstance(v, torch.FloatTensor): v=v.float()\n    return v.cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Submit to kaggle:\nif torch.cuda.is_available():\n    X_test = torch.from_numpy(X_test).float().cuda()\nelse:\n    X_test = torch.from_numpy(X_test).float()\n    \n\nlearn.freeze()\n\nyp = np.zeros((28000 ,2), dtype='int')\n\nfor i in range(0, 28000, 100):\n    prob_sum = torch.argmax(F.softmax(learn.model(X_test[i:i+100]), dim=1), dim=1).cpu().numpy()\n    yp[i:i+100] = np.array(np.vstack([np.arange(i+1, i+101), prob_sum]).T)\n    \nres = pd.DataFrame(yp, index = np.arange(len(yp)), columns=[\"ImageId\", \"Label\"])\nres.to_csv(\"submission.csv\", index=False)\nres.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}