{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_kg_hide-output":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_kg_hide-output":true},"cell_type":"code","source":"!python pytorch-xla-env-setup.py ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"!pip install pytorch_lightning --quiet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch_xla\ntorch_xla._XLAC._xla_get_devices()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport pytorch_lightning as pl\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\n\ntorch.manual_seed(100)\nnp.random.seed(100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf.iloc[:3,:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train, df_val = train_test_split(df, test_size=.1, stratify=df.label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"def toX(df):\n    return df.values.astype(np.float32).reshape(-1,1,28,28) / 127.5 - 1.\n    \nclass MnistDataLoader(object):\n    def __init__(self, df, bs):\n        self.X = toX(df.iloc[:, 1:])\n        self.y = df.values[:, 0]\n        self.bs = bs\n        self.n_batches = int(np.ceil(df.shape[0] / bs))\n    def __len__(self):\n        return self.n_batches\n    def __iter__(self):\n        m = self.y.shape[0]\n        for i in range(self.n_batches):\n            idx = np.random.randint(0, m, self.bs)\n            X = torch.tensor(self.X[idx])\n            y = torch.tensor(self.y[idx])\n            yield X, y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"class ResnetBlock(nn.Module):\n    def __init__(self, channels):\n        super(ResnetBlock, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(channels, channels, 3, 1, 1, bias=False),\n            nn.BatchNorm2d(channels),\n            nn.ReLU(),\n            nn.Conv2d(channels, channels, 3, 1, 1, bias=False),\n            nn.BatchNorm2d(channels))\n\n    def forward(self, x):        \n        x = F.relu(self.conv(x)+ x)\n        x = F.max_pool2d(x, 2)        \n        return F.dropout2d(x, .2, self.training)\n    \n    \nclass MiniResnet(nn.Module):\n    def __init__(self, channels=32):\n        super(MiniResnet, self).__init__()\n        self.backbone = nn.Sequential(\n            nn.Conv2d(1, channels, kernel_size=3, padding=1, bias=False),\n            nn.BatchNorm2d(channels),\n            nn.ReLU(),\n            ResnetBlock(channels),\n            ResnetBlock(channels)\n        )\n        self.head = nn.Sequential(\n            nn.Linear(channels*49, 256),\n            nn.ReLU(),\n            nn.Dropout(.2),\n            nn.Linear(256,10))\n        \n    def forward(self, x):\n        x = self.backbone(x)\n        dims = np.prod(x.shape[1:])\n        logits = self.head(x.view(-1, dims))\n        return logits","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training on TPUs"},{"metadata":{"trusted":true},"cell_type":"code","source":"class SimpleLogger(pl.loggers.base.DummyLogger):\n    def __init__(self):\n        super().__init__()\n        self.entries = {}\n    def log_metrics(self, metrics, step):\n        e = metrics['epoch']\n        if e in self.entries:\n            self.entries[e].update(metrics)\n        else:\n            self.entries[e] = metrics\n    def to_pandas(self):        \n        rows = [self.entries[e] for e in sorted(self.entries.keys())]\n        return pd.DataFrame(rows)\n    def plot_metrics(self, offset=0):\n        metrics = self.to_pandas()\n        names = [c for c in metrics.columns if c not in ['train_loss', 'valid_loss', 'epoch']]\n        fig, ax = plt.subplots(1, 1+len(names), figsize=(12, 3))\n        a = ax[0] if len(names) > 0 else ax\n        x = metrics.epoch.values[offset:]\n        a.plot(x, metrics.train_loss.values[offset:], label='Train')\n        a.plot(x, metrics.valid_loss.values[offset:], label='Val')\n        a.legend()\n        a.set_title('Learning curves')\n        for i, n in enumerate(names, 1):\n            ax[i].plot(x, metrics[n].values[offset:])\n            ax[i].set_title(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_kg_hide-output":true},"cell_type":"code","source":"class MnistModule(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.model = MiniResnet()\n        self.loss_fn = nn.CrossEntropyLoss()\n    def forward(self, X):\n        return self.model(X)\n    def training_step(self, batch, batch_i):\n        X, y = batch\n        h = self.model(X)\n        loss = self.loss_fn(h, y)\n        self.log('train_loss', loss, on_step=False, on_epoch=True, prog_bar=True, logger=True)\n        return loss\n    def validation_step(self, batch, batch_idx):\n        x, y = batch\n        h = self.model(x)\n        loss = self.loss_fn(h, y)\n        accuracy = (h.argmax(1) == y).float().mean()\n        self.log('valid_loss', loss, on_step=False, on_epoch=True, logger=True)\n        self.log('accuracy', accuracy, on_step=False, on_epoch=True, logger=True)\n\n    def configure_optimizers(self):\n        opt = torch.optim.Adam(self.parameters())\n        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=2)\n        return [opt], [{ 'scheduler': scheduler, 'monitor': 'valid_loss'}]\n    \n    def predict(self, X):\n        with torch.no_grad():\n            self.eval()\n            logits = self.model(X)\n            return F.softmax(logits, 1)\n\nbatch_size = 512\n\ntrain_dl = MnistDataLoader(df_train, batch_size)\nval_dl = MnistDataLoader(df_val, batch_size)\n\nmodule = MnistModule()\ntrainer = pl.Trainer(max_epochs=30, tpu_cores=1,logger=SimpleLogger())\ntrainer.fit(module, train_dl, val_dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainer.logger.plot_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_df = trainer.logger.to_pandas()\nres_df.tail(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze Errors"},{"metadata":{"trusted":true},"cell_type":"code","source":"x = torch.tensor(val_dl.X, device=module.device)\nh = module.predict(x).cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat = h.argmax(1)\n(y_hat == val_dl.y).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask_mistakes = y_hat != val_dl.y\nmask_mistakes.sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"h_mistakes = h[mask_mistakes]\ny_mistakes = val_dl.y[mask_mistakes]\nX_mistakes = val_dl.X[mask_mistakes]\ndigits, n_mistakes = np.unique(y_mistakes, return_counts=True)\nplt.bar(digits, n_mistakes)\nn_mistakes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_mistakes(X_mistakes, y_mistakes, h_mistakes, n = 5):\n    fig, subs = plt.subplots(1, n, figsize=(12,2))\n    for a in subs:\n        idx = np.random.randint(0, X_mistakes.shape[0])\n        digit = X_mistakes[idx][0]\n        a.imshow(digit, cmap='gray')\n        a.axis('off')\n        label = y_mistakes[idx]\n        p_label = h_mistakes[idx].argmax()\n        score = h_mistakes[idx, label]\n        p_score = h_mistakes[idx, p_label]\n        descr = f'Label: {label} ({score:.2f})\\nPredicted {p_label} ({p_score:.2f})'\n        a.set_title(descr)\n\nshow_mistakes(X_mistakes, y_mistakes, h_mistakes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ndf_sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = torch.tensor(toX(df_test), device = module.device)\n%time preds = module.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sub['Label'] = preds.argmax(1).cpu().numpy()\ndf_sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm *.whl\n!rm *.py\n!rm *.ckpt\n!ls","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}