{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pathlib\nimport time\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport torch\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import TSNE\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"start = time.time()\ntrain_fp = pathlib.Path('/kaggle/input/digit-recognizer/train.csv')\ntrain_x, train_y = [], []\nwith open(train_fp, 'r') as fh:\n    next(fh)\n    for line in fh:\n        label, *image = line.split(',')\n        train_x.append([int(p) for p in image])\n        train_y.append(int(label))\ntrain_x = np.array(train_x).reshape(-1, 28, 28)\ntrain_y = np.array(train_y)\nprint(f'Loaded training data in {time.time() - start:.1f}s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, counts = np.unique(train_y, return_counts=True)\ncounts = [*counts, sum(counts)]\nindex = [*list(range(10)), 'total']\ncolumns = ['']\ndisplay(pd.DataFrame(counts, index=index, columns=columns).T)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 12))\nfor i in range(6**2):\n    plt.subplot(6, 6, i + 1)\n    plt.imshow(train_x[i], cmap=plt.cm.gray)\n    plt.xlabel(str(train_y[i]))\n    plt.xticks([])\n    plt.yticks([])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"centroids = [train_x[train_y == i].mean(axis=0) for i in range(10)]\nplt.figure(figsize=(12, 4))\nfor i, centroid in enumerate(centroids, 1):\n    plt.subplot(2, 5, i)\n    plt.imshow(centroid, cmap=plt.cm.gray)\n    plt.xlabel(str(i))\n    plt.xticks([])\n    plt.yticks([])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nfor i, alg in enumerate([PCA(2), TSNE(2)], 1):\n    x_reduced = alg.fit_transform(train_x[:1000].reshape(-1, 784))\n    y_reduced = train_y[:1000]\n    data = [x_reduced[y_reduced == i] for i in range(10)]\n    plt.subplot(1, 2, i)\n    for j, points in enumerate(data):\n        plt.scatter(points[:, 0], points[:, 1], s=16, alpha=0.7, label=str(j))\n    plt.title(f'1000 images from training data using {type(alg).__name__}')\n    plt.legend()\n    plt.xticks([])\n    plt.yticks([])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ConvClassifier(torch.nn.Module):\n\n    def __init__(self):\n        super().__init__()\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        print(f'Using device: {self.device}')\n\n        self.layers1 = torch.nn.ModuleList([\n            torch.nn.Conv2d(1, 32, 5),\n            torch.nn.ReLU(),\n            torch.nn.MaxPool2d(2),\n            torch.nn.Dropout2d(0.25),\n            torch.nn.Conv2d(32, 64, 5),\n            torch.nn.ReLU(),\n            torch.nn.MaxPool2d(2),\n            torch.nn.Dropout2d(0.25),\n            torch.nn.Conv2d(64, 128, 1),\n            torch.nn.ReLU(),\n            torch.nn.MaxPool2d(2)])\n        self.layers2 = torch.nn.ModuleList([\n            torch.nn.Dropout(0.25),\n            torch.nn.Linear(512, 150),\n            torch.nn.ReLU(),\n            torch.nn.Dropout(0.25),\n            torch.nn.Linear(150, 10)])\n        n_params = sum([np.prod(p.size()) for p in self.parameters() if p.requires_grad])\n        print(f'Number of parameters: {n_params}')\n\n        self.loss = torch.nn.functional.cross_entropy\n        self.optimizer = torch.optim.Adam(self.parameters())\n\n    def fit(self, train_x, train_y, val_x, val_y, epochs=50, batch_size=250):\n        start = time.time()\n        self.to(self.device)\n        train_loader = self._build_loader_xy(\n            train_x, train_y, batch_size=batch_size, shuffle=True)\n        val_loader = self._build_loader_xy(val_x, val_y, batch_size=1000, shuffle=False)\n        history = {'epoch': [], 'loss': [], 'acc': [], 'val_loss': [], 'val_acc': []}\n        for epoch in range(1, epochs + 1):\n            self.train()\n            start_time = time.time()\n            loss, acc = 0, 0\n            for x, y in train_loader:\n                # Batch training step\n                self.optimizer.zero_grad()\n                x, y = x.to(self.device), y.to(self.device)\n                y_pred = self(x)\n                loss_ = self.loss(y_pred, y)\n                loss_.backward()\n                self.optimizer.step()\n                # Calculate metrics for batch\n                loss += float(loss_) * len(x)\n                acc += int((y_pred.argmax(dim=1) == y).sum())\n            # Calculate metrics for epoch\n            loss /= len(train_loader.dataset)\n            acc /= len(train_loader.dataset)\n            val_loss, val_acc = self._evaluate(val_loader)\n            duration = time.time() - start_time\n            # Save training history\n            history['epoch'].append(epoch)\n            history['loss'].append(loss)\n            history['acc'].append(acc)\n            history['val_loss'].append(val_loss)\n            history['val_acc'].append(val_acc)\n            print(\n                f'[{epoch:{len(str(epochs))}}/{epochs}] {duration:.1f}s'\n                f' - loss: {loss:.4f} - acc: {acc:.4f}'\n                f' - val_loss: {val_loss:.4f} - val_acc: {val_acc:.4f}')\n        return history\n\n    def predict(self, x, batch_size=1000):\n        self.to(self.device)\n        data_loader = self._build_loader_x(x, batch_size=batch_size, shuffle=False)\n        self.eval()\n        preds = []\n        with torch.no_grad():\n            for x, in data_loader:\n                x = x.to(self.device)\n                preds.extend(self(x).argmax(dim=1).cpu())\n        return np.array(preds)\n\n    def forward(self, x):\n        x = x.view(x.size()[0], 1, *x.size()[1:])\n        for layer in self.layers1:\n            x = layer(x)\n        x = x.view(x.size()[0], -1)\n        for layer in self.layers2:\n            x = layer(x)\n        return x\n\n    def _evaluate(self, data_loader):\n        self.to(self.device)\n        self.eval()\n        loss, acc = 0, 0\n        with torch.no_grad():\n            for x, y in data_loader:\n                x, y = x.to(self.device), y.to(self.device)\n                y_pred = self(x)\n                loss += float(self.loss(y_pred, y)) * len(x)\n                acc += int((y_pred.argmax(dim=1) == y).sum())\n        loss /= len(data_loader.dataset)\n        acc /= len(data_loader.dataset)\n        return loss, acc\n\n    @staticmethod\n    def _build_loader_x(x, batch_size, shuffle):\n        return torch.utils.data.DataLoader(\n            torch.utils.data.TensorDataset(torch.FloatTensor(x) / 255),\n            batch_size=batch_size, shuffle=shuffle)\n\n    @staticmethod\n    def _build_loader_xy(x, y, batch_size, shuffle):\n        return torch.utils.data.DataLoader(\n            torch.utils.data.TensorDataset(torch.FloatTensor(x) / 255, torch.LongTensor(y)),\n            batch_size=batch_size, shuffle=shuffle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ConvClassifier()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x, val_x, train_y, val_y = train_test_split(\n    train_x, train_y, test_size=2000, random_state=1234)\nhistory = model.fit(train_x, train_y, val_x, val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 5))\nfor i, (name, key) in enumerate([('accuracy', 'acc'), ('loss', 'loss')], 1):\n    plt.subplot(1, 2, i)\n    plt.plot(history['epoch'], history[key], label=f'training {name}')\n    plt.plot(history['epoch'], history[f'val_{key}'], label=f'validation {name}')\n    plt.title(f'{name} during training')\n    plt.legend()\n    plt.xlabel('epoch')\n    plt.ylabel(name)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12, 6))\ncm = confusion_matrix(val_y, model.predict(val_x))\nthreshold = np.max(cm) / 2\nplt.imshow(cm, cmap=plt.cm.binary)\nfor i in range(10):\n    for j in range(10):\n        plt.text(\n            j, i, cm[i, j], color='white' if cm[i, j] > threshold else 'black',\n            horizontalalignment='center', verticalalignment='center')\nplt.title(f'confusion matrix')\nplt.xlabel('predicted label')\nplt.xticks(range(10), range(10))\nplt.ylabel('true label')\nplt.yticks(range(10), range(10))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start = time.time()\ntest_fp = pathlib.Path('/kaggle/input/digit-recognizer/test.csv')\ntest_x = []\nwith open(test_fp, 'r') as fh:\n    next(fh)\n    for line in fh:\n        image = line.split(',')\n        test_x.append([int(p) for p in image])\ntest_x = np.array(test_x).reshape(-1, 28, 28)\nprint(f'Loaded test data in {time.time() - start:.1f}s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open('/kaggle/working/digit-recognizer_submission.csv', 'w') as fh:\n    fh.write('ImageId,Label\\n')\n    for i, label in enumerate(model.predict(test_x), 1):\n        fh.write(f'{i},{label}\\n')\nprint('Done')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}