{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom tqdm.notebook import tnrange\nfrom sklearn.model_selection import train_test_split\nfrom torchvision import models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = np.random.randint(0, df.shape[0])\nlabel = df.iloc[idx, 0].item()\nimg = df.iloc[idx, 1:].values.reshape(28,-1)\nplt.imshow(img, cmap='gray')\nplt.title('Digit: %d' % label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train, df_val = train_test_split(df, test_size = .1, stratify=df.label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = 'cuda'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Quick and dirty data loader, no transformations, just scale the images and randomly pick the batches"},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_to_X(df):\n    return df.values.astype(np.float32).reshape(-1, 1, 28,28) / 127.5 - 1    \n\ndef t(data):\n    return torch.tensor(data, device=DEVICE)\n\nclass DataLoader(object):\n    def __init__(self, df, bs):\n        self.X = df_to_X(df.iloc[:, 1:])\n        self.y = df.label.values\n        self.bs = bs\n        self.m = df.shape[0]\n        self.n_batch = int(np.ceil(self.m / bs))\n    def __len__(self):\n        return self.n_batch\n    def __call__(self):\n        for i in range(self.n_batch):\n            idx = np.random.randint(0, self.m, size=self.bs)\n            X = self.X[idx]\n            y = self.y[idx]\n            yield t(X), t(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnetBlock(nn.Module):\n    def __init__(self, channels):\n        super(ResnetBlock, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(channels, channels, 3, 1, 1, bias=False),\n            nn.BatchNorm2d(channels),\n            nn.ReLU(),\n            nn.Conv2d(channels, channels, 3, 1, 1, bias=False),\n            nn.BatchNorm2d(channels))\n\n    def forward(self, x):\n        return F.relu(self.conv(x) + x)\n    \nclass Resnet1(nn.Module):\n    def __init__(self, channels=48):\n        super(Resnet1, self).__init__()\n        self.resnet1 = nn.Sequential(\n            nn.Conv2d(1, channels, kernel_size=5, stride=2, padding=2, bias=False),\n            nn.BatchNorm2d(channels),\n            nn.ReLU(),\n            nn.MaxPool2d(3,2,1),\n            ResnetBlock(channels))\n        self.head = nn.Linear(channels*49, 10)\n    def forward(self, x):\n        x = self.resnet1(x)\n        dims = np.prod(x.shape[1:])\n        logits = self.head(x.view(-1, dims))\n        return F.log_softmax(logits, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_batch(model, optim, X, y):\n    model.train()\n    optim.zero_grad()\n    h = model(X)\n    loss = F.nll_loss(h, y)\n    loss.backward()\n    optim.step()\n    return loss.item()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 512\ntrain_dl = DataLoader(df_train, batch_size)\nval_dl = DataLoader(df_val, batch_size)\nmodel = Resnet1().to(DEVICE)\noptimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://sgugger.github.io/how-do-you-find-a-good-learning-rate.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"def smooth_loss(moving_avg, value, batch_count, beta=.98):\n    moving_avg = beta * moving_avg + (1-beta) * value \n    return moving_avg, moving_avg / (1 - beta**batch_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def find_lr(start, end):\n    n_batches = len(train_dl)\n    log_lrs = np.linspace(start, end, n_batches) \n    losses = np.zeros(n_batches)\n    moving_avg = 0\n    min_loss = None\n    for i, batch in enumerate(train_dl()):\n        # Pick a learing rate as 10^-8, 10^-7, ...\n        lr =  10 ** log_lrs[i]\n        optimizer.param_groups[0]['lr'] = lr\n        # Perform back prop and gradient update with this learning rate\n        loss = train_batch(model, optimizer, *batch)\n        moving_avg, loss = smooth_loss(moving_avg, loss, i+1)\n        losses[i] = loss\n        \n        # Check if loss is too high - if so, stop here \n        if i == 0:\n            min_loss = loss\n        elif loss < min_loss:\n            min_loss = loss\n        elif loss > min_loss*2:\n            print(\"Loss exploded\")\n            break\n    \n    lrs = log_lrs[:i]\n    losses = losses[:i]    \n    return lrs, losses\n\nlrs, losses = find_lr(-8,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = 1e-2\nplt.plot(lrs, losses)\nplt.axvline(np.log10(lr),  color='r')\nplt.title(r'Best learing rate is around $10^{-2}$', fontsize=20)\n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def validate(model, dl):\n    model.eval()\n    with torch.no_grad():\n        all_h, all_y = [], []\n        for X, y in dl():\n            h = model(X)\n            all_h.append(h)\n            all_y.append(y)\n        h = torch.cat(all_h, 0)\n        y = torch.cat(all_y, 0)\n        val_loss = F.nll_loss(h, y).item() / dl.n_batch\n        accuracy = (torch.argmax(h, 1) == y).float().mean().item()\n    return val_loss, accuracy\n\ndef train(model, opt, train_dl, val_dl, E=1):\n    metrics = np.zeros([E, 3])    \n    with tnrange(E) as pbar:\n        for e in pbar:\n            model.train()\n            train_loss = 0\n            for X, y in train_dl():\n                batch_loss = train_batch(model, opt, X, y)\n                train_loss += batch_loss / train_dl.n_batch\n            val_loss, accuracy = validate(model, val_dl)\n            metrics[e] = [train_loss, val_loss, accuracy]\n            pbar.set_postfix(train_loss = f'{train_loss:.3f}')\n    return metrics","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# The recommended learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_metrics(metrics):\n    fig, ax = plt.subplots(1,2, figsize=(12,3))\n    x = np.arange(1,metrics.shape[0]+1)\n    ax[0].plot(x, metrics[:, 0], label='Train Loss')\n    ax[0].plot(x, metrics[:, 1], label='Validation Loss')\n    ax[1].plot(x, metrics[:, 2])\n    ax[0].set_title(\"Learning curves\")\n    ax[0].legend()\n    ax[1].set_title('Accuracy (Val set)')\n\nmodel = Resnet1().to(DEVICE)\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)\nmetrics = train(model, optimizer, train_dl, val_dl, 5)    \nplot_metrics(metrics)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Smaller learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Resnet1().to(DEVICE)\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)\nmetrics = train(model, optimizer, train_dl, val_dl, 5)    \nplot_metrics(metrics)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Larger learning rate"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Resnet1().to(DEVICE)\noptimizer = torch.optim.SGD(model.parameters(), lr=5e-1, momentum=0.9)\nmetrics = train(model, optimizer, train_dl, val_dl, 5)    \nplot_metrics(metrics)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Resnet1().to(DEVICE)\noptimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)\nmetrics = train(model, optimizer, train_dl, val_dl, 20)    \nprint(\"Validation set accuracy:\", metrics[-1, -1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = t(df_to_X(df_test))\nh = model(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = h.argmax(1).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"N = 3\nidx = np.random.randint(0, labels.shape[0], N)\n\nfig, ax = plt.subplots(1, N, figsize=(15,3))\nfor i, a in enumerate(ax):\n    a.axes.get_xaxis().set_visible(False)\n    a.axes.get_yaxis().set_visible(False)\n    img = df_test.iloc[idx[i]].values.reshape(28,28)\n    label = labels[idx[i]]\n    a.imshow(img, cmap='gray')\n    a.set_title(f'We predict this is: {label}', fontsize=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")\ndf_submission.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['Label'] = labels\ndf_submission.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv('submission.csv', index=False)\nwith open('submission.csv') as f:\n    for i in range(3):\n        l = f.readline()\n        print(l.strip())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}