{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-21T04:39:31.382683Z","iopub.execute_input":"2022-01-21T04:39:31.383849Z","iopub.status.idle":"2022-01-21T04:39:31.420309Z","shell.execute_reply.started":"2022-01-21T04:39:31.383714Z","shell.execute_reply":"2022-01-21T04:39:31.419098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:31.423159Z","iopub.execute_input":"2022-01-21T04:39:31.423537Z","iopub.status.idle":"2022-01-21T04:39:36.051801Z","shell.execute_reply.started":"2022-01-21T04:39:31.423489Z","shell.execute_reply":"2022-01-21T04:39:36.050707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X, y = df_train.drop(columns = [\"label\"]), df_train[\"label\"]","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:36.053309Z","iopub.execute_input":"2022-01-21T04:39:36.053712Z","iopub.status.idle":"2022-01-21T04:39:36.176043Z","shell.execute_reply.started":"2022-01-21T04:39:36.053670Z","shell.execute_reply":"2022-01-21T04:39:36.174922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:36.178833Z","iopub.execute_input":"2022-01-21T04:39:36.179147Z","iopub.status.idle":"2022-01-21T04:39:36.212930Z","shell.execute_reply.started":"2022-01-21T04:39:36.179104Z","shell.execute_reply":"2022-01-21T04:39:36.212001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:36.214586Z","iopub.execute_input":"2022-01-21T04:39:36.215038Z","iopub.status.idle":"2022-01-21T04:39:37.255663Z","shell.execute_reply.started":"2022-01-21T04:39:36.214995Z","shell.execute_reply":"2022-01-21T04:39:37.254703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2)\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.1)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:37.258676Z","iopub.execute_input":"2022-01-21T04:39:37.259231Z","iopub.status.idle":"2022-01-21T04:39:37.765308Z","shell.execute_reply.started":"2022-01-21T04:39:37.259187Z","shell.execute_reply":"2022-01-21T04:39:37.764279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, X_val.shape, X_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:37.767891Z","iopub.execute_input":"2022-01-21T04:39:37.768478Z","iopub.status.idle":"2022-01-21T04:39:37.779237Z","shell.execute_reply.started":"2022-01-21T04:39:37.768433Z","shell.execute_reply":"2022-01-21T04:39:37.778187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch.utils.data import Dataset, DataLoader\n\n\nclass MNISTDataset(Dataset):\n    def __init__(self, X, y):\n        self.X = torch.Tensor(X.values.reshape(-1, 1, 28, 28))\n        self.y = torch.LongTensor(y.values)\n    def __len__(self):\n        return len(self.X)\n    def __getitem__(self, idx):\n        return self.X[idx], self.y[idx]\n    \ndatasets = {}\ndatasets[\"train\"] = MNISTDataset(X_train, y_train)\ndatasets[\"val\"] = MNISTDataset(X_val, y_val)\ndatasets[\"test\"] = MNISTDataset(X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:37.780921Z","iopub.execute_input":"2022-01-21T04:39:37.781284Z","iopub.status.idle":"2022-01-21T04:39:39.564910Z","shell.execute_reply.started":"2022-01-21T04:39:37.781237Z","shell.execute_reply":"2022-01-21T04:39:39.563855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_x, sample_y = datasets[\"train\"][0]\nsample_x, sample_y","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:39.566815Z","iopub.execute_input":"2022-01-21T04:39:39.567137Z","iopub.status.idle":"2022-01-21T04:39:39.638274Z","shell.execute_reply.started":"2022-01-21T04:39:39.567093Z","shell.execute_reply":"2022-01-21T04:39:39.637067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataloaders = {}\nBATCH_SIZE = 64\ndataloaders[\"train\"] = DataLoader(datasets[\"train\"], batch_size=BATCH_SIZE, shuffle=True)\ndataloaders[\"val\"] = DataLoader(datasets[\"val\"], batch_size=BATCH_SIZE, shuffle=False)\ndataloaders[\"test\"] = DataLoader(datasets[\"test\"], batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:39.642712Z","iopub.execute_input":"2022-01-21T04:39:39.643003Z","iopub.status.idle":"2022-01-21T04:39:39.649848Z","shell.execute_reply.started":"2022-01-21T04:39:39.642973Z","shell.execute_reply":"2022-01-21T04:39:39.648706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\n\nclass MNISTModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        in_channels = [1, 32, 64]\n        out_channels = [32, 64, 128]\n        kernel_sizes = [3, 3, 3]\n        strides = [1, 1, 1]\n        paddings = [1, 1, 1]\n        \n        self.layer0 = nn.Sequential(\n            nn.Conv2d(\n                in_channels = in_channels[0], \n                out_channels = out_channels[0], \n                kernel_size = kernel_sizes[0], \n                stride = strides[0], \n                padding = paddings[0]),\n            nn.Conv2d(\n                in_channels = out_channels[0], \n                out_channels = out_channels[0], \n                kernel_size = kernel_sizes[0], \n                stride = strides[0], \n                padding = paddings[0]),\n            nn.BatchNorm2d(out_channels[0]),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2),\n        )\n        \n        self.layer1 = nn.Sequential(\n            nn.Conv2d(\n                in_channels = in_channels[1], \n                out_channels = out_channels[1], \n                kernel_size = kernel_sizes[1], \n                stride = strides[1], \n                padding = paddings[1]),\n            nn.Conv2d(\n                in_channels = out_channels[1], \n                out_channels = out_channels[1], \n                kernel_size = kernel_sizes[1], \n                stride = strides[1], \n                padding = paddings[1]),\n            nn.BatchNorm2d(out_channels[1]),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size = 2),\n        )\n        \n        self.layer2 = nn.Sequential(\n            nn.Conv2d(\n                in_channels = in_channels[2], \n                out_channels = out_channels[2], \n                kernel_size = kernel_sizes[2], \n                stride = strides[2], \n                padding = paddings[2]),\n            nn.Conv2d(\n                in_channels = out_channels[2], \n                out_channels = out_channels[2], \n                kernel_size = kernel_sizes[2], \n                stride = strides[2], \n                padding = paddings[2]),\n            nn.BatchNorm2d(out_channels[2]),\n            nn.ReLU(),\n        )\n        \n        self.linear0 = nn.Linear(128*7*7, 512)\n        self.linear1 = nn.Linear(512, 256)\n        self.linear2 = nn.Linear(256, 10)\n        \n        \n    def forward(self, x):\n        x = self.layer0(x)\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = x.reshape(-1, 128*7*7)\n        x = self.linear0(x)\n        x = self.linear1(x)\n        out = self.linear2(x)\n        return out","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:39.652274Z","iopub.execute_input":"2022-01-21T04:39:39.652996Z","iopub.status.idle":"2022-01-21T04:39:39.676770Z","shell.execute_reply.started":"2022-01-21T04:39:39.652946Z","shell.execute_reply":"2022-01-21T04:39:39.675648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:39.680206Z","iopub.execute_input":"2022-01-21T04:39:39.681199Z","iopub.status.idle":"2022-01-21T04:39:39.744968Z","shell.execute_reply.started":"2022-01-21T04:39:39.681150Z","shell.execute_reply":"2022-01-21T04:39:39.743872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MNISTModel()\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:39.747304Z","iopub.execute_input":"2022-01-21T04:39:39.747617Z","iopub.status.idle":"2022-01-21T04:39:43.147537Z","shell.execute_reply.started":"2022-01-21T04:39:39.747585Z","shell.execute_reply":"2022-01-21T04:39:43.146534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_x, sample_y = next(iter(dataloaders[\"train\"]))","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:43.152608Z","iopub.execute_input":"2022-01-21T04:39:43.153360Z","iopub.status.idle":"2022-01-21T04:39:43.182689Z","shell.execute_reply.started":"2022-01-21T04:39:43.153317Z","shell.execute_reply":"2022-01-21T04:39:43.181852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import optim\nEPOCH = 100\noptimizer = optim.Adam(model.parameters(), lr = 1e-03)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10, eta_min=0)\nloss_fn = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:43.187621Z","iopub.execute_input":"2022-01-21T04:39:43.190047Z","iopub.status.idle":"2022-01-21T04:39:43.199715Z","shell.execute_reply.started":"2022-01-21T04:39:43.190007Z","shell.execute_reply":"2022-01-21T04:39:43.198497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\n\n\nmin_val_loss = 10000000\n\nPATH = \"best_model.pth\"\n\nfor e in range(1, EPOCH+1):\n    print(f\"EPOCH {e}\")\n    \n    model.train()\n    train_loss = 0\n    for X, y in tqdm(dataloaders[\"train\"]):\n        X, y = X.to(device), y.to(device)\n        y_pred = model(X)\n        \n        optimizer.zero_grad()\n        loss = loss_fn(y_pred, y)\n        loss.backward()\n        optimizer.step()\n        \n        train_loss += loss.item()\n    train_loss /= len(dataloaders[\"train\"])\n    \n    \n    with torch.no_grad():\n        model.eval()\n        val_loss = 0.0\n        for X, y in tqdm(dataloaders[\"val\"]):\n            X, y = X.to(device), y.to(device)\n            y_pred = model(X)\n            loss = loss_fn(y_pred, y)\n            val_loss += loss.item()\n        val_loss /= len(dataloaders[\"val\"])\n    \n    print(f\"train loss: {train_loss}, val loss: {val_loss}\")\n    \n    if val_loss < min_val_loss:\n        print(\"saving best model...\")\n        min_val_loss = val_loss\n        torch.save(model.state_dict(), PATH)\n        \n    scheduler.step()\n    print(\"lr: \", optimizer.param_groups[0]['lr'])","metadata":{"execution":{"iopub.status.busy":"2022-01-21T04:39:52.139771Z","iopub.execute_input":"2022-01-21T04:39:52.140157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model = MNISTModel()\nbest_model.load_state_dict(torch.load(PATH))\nbest_model.to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.eval()\n\nwith torch.no_grad():\n    correct = 0\n    total = 0\n    for X, y in tqdm(dataloaders[\"test\"]):\n        X, y = X.to(device), y.to(device)\n        y_pred = best_model(X)\n        y_pred = torch.argmax(y_pred, dim = 1)\n        \n        correct += torch.sum(y_pred == y)\n        total += len(y)\n    print(correct/total)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_final = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_final = torch.Tensor(X_final.values.reshape(-1, 1, 28, 28)).to(device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_final = []\n\nfor i in range(0, len(X_final), 1000):\n    if i + 1000 >= len(X_final):\n        small_y_final = torch.argmax(best_model(X_final[i:]), dim = 1)\n    else:\n        small_y_final = torch.argmax(best_model(X_final[i:i+1000]), dim = 1)\n    y_final.append(small_y_final)\n    \ny_final = torch.cat(y_final)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission[\"Label\"] = y_final.to(\"cpu\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission.to_csv(\"submission4.csv\", index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}