{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from mxnet import autograd, np, npx, init\nfrom mxnet.gluon import nn\nfrom mxnet import gluon\nfrom mxnet.gluon.data import ArrayDataset, DataLoader\nfrom mxnet.image import imread\nfrom mxnet.metric import Accuracy\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom mxnet.gluon.data.vision import transforms\nfrom sklearn.preprocessing import LabelEncoder\nimport seaborn as sns\nfrom IPython import display\n\nnpx.set_np()\nctx = npx.gpu()\nsns.set_theme()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/dog-breed-identification/labels.csv')\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define transforms for training and test\nFor the training set, we generating similar but distinct images by flipping and cropping.\nFor both sets we transforme image of shape (H, W, C) to (C, H, W), and normilize pixels.","metadata":{}},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.RandomResizedCrop(331),\n    transforms.RandomFlipLeftRight(),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.485, 0.456, 0.406),\n                         std=(0.229, 0.224, 0.255))\n])\n\ntest_transform = transforms.Compose([\n    transforms.Resize(350),\n    transforms.CenterCrop(331),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=(0.485, 0.456, 0.406),\n                         std=(0.229, 0.224, 0.255))\n])\n\nclass ImageLoader():\n    def __init__(self, is_train=False):\n        self._is_train = is_train\n        \n    def __call__(self, image_name, label=None):\n        if self._is_train:\n            folder = 'train'\n            transform = train_transform\n        else:\n            folder = 'test'\n            transform = test_transform\n        \n        path = f'../input/dog-breed-identification/{folder}/{image_name}.jpg'\n        \n        if label is None:\n            return transform(imread(path))\n        else:\n            return transform(imread(path)), label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define DenseNet from scratch","metadata":{}},{"cell_type":"code","source":"class ConvBlock(nn.Block):\n    def __init__(self, num_channels, **kwargs):\n        super().__init__(**kwargs)\n        self.net = nn.Sequential()\n        self.net.add(\n            nn.BatchNorm(),\n            nn.Activation('relu'),\n            nn.Conv2D(num_channels, kernel_size=3, padding=1)\n        )\n    \n    def forward(self, X):\n        return self.net(X)\n\n\nclass DenseBlock(nn.Block):\n    def __init__(self, num_convs, num_channels, **kwargs):\n        super().__init__(**kwargs)\n        self.net = nn.Sequential()\n        \n        for _ in range(num_convs):\n            self.net.add(ConvBlock(num_channels))\n        \n    def forward(self, X):\n        for blk in self.net:\n            Y = blk(X)\n            X = np.concatenate((X, Y), axis=1)\n        \n        return X\n\n\nclass TransitionBlock(nn.Block):\n    def __init__(self, num_channels, **kwargs):\n        super().__init__(**kwargs)\n        self.net = nn.Sequential()\n        self.net.add(\n            nn.BatchNorm(),\n            nn.Activation('relu'),\n            nn.Conv2D(num_channels, kernel_size=1),\n            nn.AvgPool2D(pool_size=2, strides=2)\n        )\n        \n    def forward(self, X):\n        return self.net(X)\n        \n\nclass DenseNet(nn.Block):\n    def __init__(self, **kwargs):\n        super().__init__(**kwargs)\n        self.net = nn.Sequential()\n        self.net.add(\n            nn.Conv2D(64, kernel_size=7, strides=2, padding=3),\n            nn.BatchNorm(),\n            nn.Activation('relu'),\n            nn.MaxPool2D(pool_size=3, strides=2, padding=1)\n        )\n        \n        num_channels, growth_rate = 64, 32\n        \n        for i in range(4):\n            self.net.add(DenseBlock(4, growth_rate))\n            num_channels += 4 * growth_rate\n            if i != 3:\n                num_channels //= 2\n                self.net.add(TransitionBlock(num_channels))\n            \n        self.net.add(\n            nn.BatchNorm(),\n            nn.Activation('relu'),\n            nn.GlobalAvgPool2D(),\n            nn.Dense(120)\n        )\n    \n    def forward(self, X):\n        return self.net(X)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"net = DenseNet()\nnet.initialize(init=init.Xavier(), ctx=ctx, force_reinit=True)\n\nplt.ion()\nfigure, ax = plt.subplots(figsize=(8, 6))\n\nlr, num_epochs, batch_size = 0.4, 300, 32\nX = df['id']\ny = df['breed']\ncount = len(y)\nle = LabelEncoder()\nle.fit(y.unique())\ntrain_ds =  ArrayDataset(X.values, np.array(le.transform(y), ctx=ctx, dtype=np.float32))\ntrain_dl = DataLoader(train_ds.transform(ImageLoader(is_train=True)), batch_size=batch_size)\nloss = gluon.loss.SoftmaxCrossEntropyLoss()\ntrainer = gluon.Trainer(net.collect_params(), 'sgd',\n                        {'learning_rate': lr})\n\nerrors = []\nacc = Accuracy()\n\nfor epoch in range(num_epochs):\n    err = 0\n    for X, y in train_dl:\n        X, y = X.as_in_context(ctx), y.as_in_context(ctx)\n    \n        with autograd.record():\n            y_hat = net(X)\n            l = loss(y_hat, y)\n        \n        l.backward()\n        err += l.sum()\n        trainer.step(batch_size)\n\n    errors.append(float(err) / count)\n    ax.cla()\n    ax.plot([i for i in range(epoch + 1)], errors, 'r')\n    plt.title('Learning Curve')\n    plt.xlabel('Epoch')\n    plt.ylabel('Error')\n    plt.xlim(left=0)\n    display.display(figure)\n    display.clear_output(wait=True)   ","metadata":{"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.save_parameters('net.params')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make a prediction","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv('../input/dog-breed-identification/sample_submission.csv')\n\nfor index, row in submission.iterrows():\n    image = ImageLoader()(row.iloc[0]).as_in_context(ctx).reshape((1, 3, 331, 331))\n    y_hat = net(image)\n    y_hat = npx.softmax(y_hat).asnumpy().reshape((120,))\n    submission.iloc[index, 1:] = y_hat","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}