{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-11T19:00:36.708236Z","iopub.execute_input":"2021-10-11T19:00:36.708641Z","iopub.status.idle":"2021-10-11T19:00:36.720673Z","shell.execute_reply.started":"2021-10-11T19:00:36.708551Z","shell.execute_reply":"2021-10-11T19:00:36.719667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install rich","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:36.722978Z","iopub.execute_input":"2021-10-11T19:00:36.723335Z","iopub.status.idle":"2021-10-11T19:00:42.603883Z","shell.execute_reply.started":"2021-10-11T19:00:36.723301Z","shell.execute_reply":"2021-10-11T19:00:42.602940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:42.605882Z","iopub.execute_input":"2021-10-11T19:00:42.606236Z","iopub.status.idle":"2021-10-11T19:00:43.035513Z","shell.execute_reply.started":"2021-10-11T19:00:42.606196Z","shell.execute_reply":"2021-10-11T19:00:43.034309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def uniform_split_dev_df(raw_dev_df, test_size=0.2, random_state=42):\n    n_rare_class = raw_dev_df['label'].value_counts().min()\n    classes = raw_dev_df['label'].unique()\n    classes_df = [raw_dev_df[raw_dev_df['label']==c].sample(n=n_rare_class, random_state=random_state) for c in classes]\n    splitted_classes = [train_test_split(df, test_size=test_size, random_state=random_state) for df in classes_df]\n    train_df = pd.concat([l[0] for l in splitted_classes], axis=0)\n    valid_df = pd.concat([l[1] for l in splitted_classes], axis=0)\n    return train_df, valid_df","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:43.040197Z","iopub.execute_input":"2021-10-11T19:00:43.042238Z","iopub.status.idle":"2021-10-11T19:00:43.050502Z","shell.execute_reply.started":"2021-10-11T19:00:43.042192Z","shell.execute_reply":"2021-10-11T19:00:43.049352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"raw_dev_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nraw_test_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nraw_train_df, raw_valid_df = uniform_split_dev_df(raw_dev_df, test_size=0.2, random_state=42)\n\nraw_x_train = raw_train_df.drop(columns=['label'])\nraw_y_train = raw_train_df['label']\n\nraw_x_valid = raw_valid_df.drop(columns=['label'])\nraw_y_valid = raw_valid_df['label']","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:43.052097Z","iopub.execute_input":"2021-10-11T19:00:43.052726Z","iopub.status.idle":"2021-10-11T19:00:48.054740Z","shell.execute_reply.started":"2021-10-11T19:00:43.052666Z","shell.execute_reply":"2021-10-11T19:00:48.053827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch import nn, optim\nimport torchvision","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:48.056112Z","iopub.execute_input":"2021-10-11T19:00:48.056446Z","iopub.status.idle":"2021-10-11T19:00:48.502689Z","shell.execute_reply.started":"2021-10-11T19:00:48.056409Z","shell.execute_reply":"2021-10-11T19:00:48.501859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:48.504844Z","iopub.execute_input":"2021-10-11T19:00:48.505516Z","iopub.status.idle":"2021-10-11T19:00:48.532529Z","shell.execute_reply.started":"2021-10-11T19:00:48.505478Z","shell.execute_reply":"2021-10-11T19:00:48.531784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNISTData:\n    def __init__(self, x_df, y_df, device, name):\n        x_array = x_df.to_numpy().reshape(-1, 1, 28, 28) # channels first\n        y_array = y_df.to_numpy()\n        self.x = torch.tensor(x_array/254, dtype=torch.float32, device=device)\n        self.y = torch.tensor(y_array, device=device)\n        self.length = self.x.shape[0]\n        self.device = device\n        self.name = name\n    def __len__(self):\n        return self.length\n    def __getitem__(self, idx):\n        return self.x[idx], self.y[idx]","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:48.535924Z","iopub.execute_input":"2021-10-11T19:00:48.536312Z","iopub.status.idle":"2021-10-11T19:00:48.543789Z","shell.execute_reply.started":"2021-10-11T19:00:48.536274Z","shell.execute_reply":"2021-10-11T19:00:48.542896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = MNISTData(raw_x_train, raw_y_train, device, 'trainset')\nvalidset = MNISTData(raw_x_valid, raw_y_valid, device, 'validset')","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:48.545803Z","iopub.execute_input":"2021-10-11T19:00:48.546297Z","iopub.status.idle":"2021-10-11T19:00:50.588310Z","shell.execute_reply.started":"2021-10-11T19:00:48.546259Z","shell.execute_reply":"2021-10-11T19:00:50.587387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.589676Z","iopub.execute_input":"2021-10-11T19:00:50.590081Z","iopub.status.idle":"2021-10-11T19:00:50.596412Z","shell.execute_reply.started":"2021-10-11T19:00:50.590043Z","shell.execute_reply":"2021-10-11T19:00:50.595437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def random_visualize(dataset):\n    print(f'Randomly visualizing an image from {dataset.name}')\n    idx = np.random.randint(0, len(dataset))\n    plt.figure(figsize=(5, 5))\n    plt.imshow(dataset[idx][0].cpu().numpy().reshape(28, 28), cmap='gray_r')\n    plt.axis('off')\n    plt.title(f'Label: {dataset[idx][1].item()}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.597889Z","iopub.execute_input":"2021-10-11T19:00:50.598444Z","iopub.status.idle":"2021-10-11T19:00:50.608324Z","shell.execute_reply.started":"2021-10-11T19:00:50.598400Z","shell.execute_reply":"2021-10-11T19:00:50.607540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random_visualize(trainset)\nrandom_visualize(validset)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.609599Z","iopub.execute_input":"2021-10-11T19:00:50.610004Z","iopub.status.idle":"2021-10-11T19:00:50.753119Z","shell.execute_reply.started":"2021-10-11T19:00:50.609968Z","shell.execute_reply":"2021-10-11T19:00:50.752188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\ntrain_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(trainset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.754445Z","iopub.execute_input":"2021-10-11T19:00:50.754820Z","iopub.status.idle":"2021-10-11T19:00:50.759849Z","shell.execute_reply.started":"2021-10-11T19:00:50.754783Z","shell.execute_reply":"2021-10-11T19:00:50.758599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom rich.progress import Progress","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.761517Z","iopub.execute_input":"2021-10-11T19:00:50.762087Z","iopub.status.idle":"2021-10-11T19:00:50.801080Z","shell.execute_reply.started":"2021-10-11T19:00:50.762036Z","shell.execute_reply":"2021-10-11T19:00:50.800226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_progressbar_description(train_loss, train_accuracy, valid_loss, valid_accuracy):\n    body = (f'TL: {train_loss:.2e}, TA: {train_accuracy:.2%}, '\n             f'VL: {valid_loss:.2e}, VA: {valid_accuracy:.2%}')    \n    return  '{' + body + '}'","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.802208Z","iopub.execute_input":"2021-10-11T19:00:50.802519Z","iopub.status.idle":"2021-10-11T19:00:50.810370Z","shell.execute_reply.started":"2021-10-11T19:00:50.802483Z","shell.execute_reply":"2021-10-11T19:00:50.809262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Trainer:\n    def __init__(\n        self, model, criterion, optimizer, scheduler, train_loader, valid_loader, device\n    ):\n        self.model = model\n        self.criterion = criterion\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.train_loader = train_loader\n        self.valid_loader = valid_loader\n        self.device = device\n        self.history = {\"train_loss\": [], \"valid_loss\": []}\n        self.__trained_epochs = 0\n\n    def __train_loop(self, progress, task):\n        self.model.train()\n        n_mini_batch = len(self.train_loader)\n        acm_loss = 0\n        progress.reset(task)\n        progress.update(task, visible=True)\n        for i, (x, y) in enumerate(self.train_loader):\n            pred = self.model(x.to(device))\n            loss = self.criterion(pred.squeeze(), y.to(device))\n            self.optimizer.zero_grad()\n            loss.backward()\n            self.optimizer.step()\n            acm_loss += loss.item()\n            msg = (\n                \"{\"\n                f't_b: \"{i+1}/{n_mini_batch}\", '\n                f\"t_l: {(acm_loss/(i+1))**0.5:.2e}\"\n                \"}\"\n            )\n            progress.update(task, description=msg, completed=i + 1)\n        progress.update(task, visible=False)\n        mean_loss = acm_loss / n_mini_batch\n        return mean_loss\n\n    @torch.no_grad()\n    def predict(self, x):\n        self.model.eval()\n        return self.model(x.to(device))\n\n    @torch.no_grad()\n    def __valid_loop(self, progress, task):\n        self.model.eval()\n        n_mini_batch = len(self.valid_loader)\n        acm_loss = 0\n        progress.reset(task)\n        progress.update(task, visible=True)\n        for i, (x, y) in enumerate(self.valid_loader):\n            pred = self.model(x.to(device))\n            loss = self.criterion(pred.squeeze(), y.to(device))\n            acm_loss += loss.item()\n            msg = (\n                \"{\"\n                f'v_b: \"{i+1}/{n_mini_batch}\", '\n                f\"v_l: {(acm_loss/(i+1))**0.5:.2e}\"\n                \"}\"\n            )\n            progress.update(task, description=msg, completed=i + 1)\n        progress.update(task, visible=False)\n        mean_loss = acm_loss / n_mini_batch\n        return mean_loss\n\n    def train(self, epochs):\n        with Progress() as progress:\n            epoch_task = progress.add_task(\"Training...\", total=epochs)\n            train_batch_task = progress.add_task(\n                \"\", total=len(self.train_loader), visible=False\n            )\n            valid_batch_task = progress.add_task(\n                \"\", total=len(self.valid_loader), visible=False\n            )\n            for epoch in range(1, epochs + 1):\n                train_loss = self.__train_loop(progress, train_batch_task)\n                valid_loss = self.__valid_loop(progress, valid_batch_task)\n                scheduler.step(train_loss)\n                self.history[\"train_loss\"].append(train_loss)\n                self.history[\"valid_loss\"].append(valid_loss)\n                self.__trained_epochs += 1\n                msg = (\n                    \"{\"\n                    f'e: \"{self.__trained_epochs}/{self.__trained_epochs + epochs - epoch}\", '\n                    f\"t_l: {train_loss**0.5:.2e}, \"\n                    f\"v_l: {valid_loss**0.5:.2e}\"\n                    \"}\"\n                )\n                progress.update(epoch_task, description=msg, completed=epoch)\n","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.813559Z","iopub.execute_input":"2021-10-11T19:00:50.814013Z","iopub.status.idle":"2021-10-11T19:00:50.834839Z","shell.execute_reply.started":"2021-10-11T19:00:50.813985Z","shell.execute_reply":"2021-10-11T19:00:50.833916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_padding(in_size, out_size, kernel_size, stride, dilation):\n    return (dilation*(kernel_size-1) - in_size + stride*(out_size-1) + 1) / 2","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.836040Z","iopub.execute_input":"2021-10-11T19:00:50.836379Z","iopub.status.idle":"2021-10-11T19:00:50.844541Z","shell.execute_reply.started":"2021-10-11T19:00:50.836347Z","shell.execute_reply":"2021-10-11T19:00:50.843510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResidualBlock(nn.Module):\n    def __init__(self, in_size, in_channels, out_channels, kernel_size=3, stride=1, dilation=1):\n        super().__init__()\n        padding = calculate_padding(in_size, in_size, kernel_size, stride, dilation)\n        if not padding.is_integer():\n            raise Exception('Invalid combination of in_size, kernel_size, stride, dilation')\n        padding = (int(padding), int(padding))\n        self.direct_path = nn.Sequential(\n            nn.Conv2d(\n                in_channels,\n                out_channels,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            ),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n            nn.Conv2d(\n                out_channels,\n                out_channels,\n                kernel_size,\n                stride=stride,\n                padding=padding,\n                dilation=dilation,\n            ),\n            nn.BatchNorm2d(out_channels),\n        )\n        self.relu = nn.ReLU()\n    def forward(self, x):\n        return self.relu(self.direct_path(x) + x)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.847093Z","iopub.execute_input":"2021-10-11T19:00:50.848103Z","iopub.status.idle":"2021-10-11T19:00:50.858408Z","shell.execute_reply.started":"2021-10-11T19:00:50.848066Z","shell.execute_reply":"2021-10-11T19:00:50.857634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MyMNISTResNet(nn.Module):\n    def __init__(self, n_layers, n_conv_channels=8):\n        super().__init__()\n        self.resblock_stack = nn.Sequential(\n            ResidualBlock(28, 1, 8, kernel_size=3),\n            *[ResidualBlock(28, 8, 8, kernel_size=3) for _ in range(n_layers-2)],\n        )\n        self.fc = nn.Linear(28*28*n_conv_channels, 10)\n    def forward(self, x):\n        x = self.resblock_stack(x)\n        x = torch.flatten(x, start_dim=1, end_dim=-1)\n        return self.fc(x)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.859664Z","iopub.execute_input":"2021-10-11T19:00:50.860138Z","iopub.status.idle":"2021-10-11T19:00:50.868892Z","shell.execute_reply.started":"2021-10-11T19:00:50.860103Z","shell.execute_reply":"2021-10-11T19:00:50.867755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MyMNISTResNet(4).to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=3e-4)\nscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, verbose=True, factor=0.5, patience=2)\ntrainer = Trainer(\n    model, \n    criterion,\n    optimizer,\n    scheduler,\n    train_loader, \n    valid_loader,\n    device,\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.870273Z","iopub.execute_input":"2021-10-11T19:00:50.870723Z","iopub.status.idle":"2021-10-11T19:00:50.885471Z","shell.execute_reply.started":"2021-10-11T19:00:50.870674Z","shell.execute_reply":"2021-10-11T19:00:50.884622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.train(10)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:00:50.886621Z","iopub.execute_input":"2021-10-11T19:00:50.887053Z","iopub.status.idle":"2021-10-11T19:02:07.193321Z","shell.execute_reply.started":"2021-10-11T19:00:50.887014Z","shell.execute_reply":"2021-10-11T19:02:07.192336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"import torch","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:02:07.194865Z","iopub.execute_input":"2021-10-11T19:02:07.195224Z","iopub.status.idle":"2021-10-11T19:02:07.199512Z","shell.execute_reply.started":"2021-10-11T19:02:07.195178Z","shell.execute_reply":"2021-10-11T19:02:07.198565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_tensor = torch.tensor(raw_test_df.to_numpy().reshape(-1, 1, 28, 28)/254, dtype=torch.float32, device=device)\n\nlogits = trainer.predict(test_tensor)\n\nsoftmax = torch.nn.LogSoftmax(dim=1)\n\npreds = softmax(logits).argmax(dim=1).cpu().numpy()\n\nsubmit_df = pd.DataFrame({'ImageId': pd.RangeIndex(start=1, stop=preds.shape[0]+1), 'Label': preds})","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:02:07.201168Z","iopub.execute_input":"2021-10-11T19:02:07.201554Z","iopub.status.idle":"2021-10-11T19:02:07.697813Z","shell.execute_reply.started":"2021-10-11T19:02:07.201514Z","shell.execute_reply":"2021-10-11T19:02:07.696900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:02:07.699133Z","iopub.execute_input":"2021-10-11T19:02:07.699504Z","iopub.status.idle":"2021-10-11T19:02:07.703762Z","shell.execute_reply.started":"2021-10-11T19:02:07.699469Z","shell.execute_reply":"2021-10-11T19:02:07.702658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"timestamp = int(datetime.now().timestamp())\nsubmit_df.to_csv(f'mnist_{timestamp}.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-11T19:02:07.705175Z","iopub.execute_input":"2021-10-11T19:02:07.705575Z","iopub.status.idle":"2021-10-11T19:02:07.759732Z","shell.execute_reply.started":"2021-10-11T19:02:07.705534Z","shell.execute_reply":"2021-10-11T19:02:07.759018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}