{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import math\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport numpy as np\nimport pandas as pd\n\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import datasets, transforms\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm.notebook import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-11-21T15:22:26.098232Z","iopub.execute_input":"2022-11-21T15:22:26.098676Z","iopub.status.idle":"2022-11-21T15:22:26.105652Z","shell.execute_reply.started":"2022-11-21T15:22:26.098641Z","shell.execute_reply":"2022-11-21T15:22:26.104394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH_SIZE = 64\nEPOCHS = 50\nEMB_SIZE = 128","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:22:26.334587Z","iopub.execute_input":"2022-11-21T15:22:26.334991Z","iopub.status.idle":"2022-11-21T15:22:26.341081Z","shell.execute_reply.started":"2022-11-21T15:22:26.334957Z","shell.execute_reply":"2022-11-21T15:22:26.339809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:22:26.569794Z","iopub.execute_input":"2022-11-21T15:22:26.570665Z","iopub.status.idle":"2022-11-21T15:22:30.928054Z","shell.execute_reply.started":"2022-11-21T15:22:26.570625Z","shell.execute_reply":"2022-11-21T15:22:30.926999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNIST(Dataset):\n    def __init__(self, X, y, transform=None):\n        super().__init__()\n        self.images = X\n        self.labels = y\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, idx):\n        image = self.images[idx]\n        label = self.labels[idx]\n        if self.transform is not None:\n            image = self.transform(image)\n        \n        return image, label\n\ndef get_loader(df, split=None):\n    X = df.drop(\"label\", axis=1).values.reshape(-1, 28, 28, 1) / 255\n    y = df[\"label\"].values\n\n    transform = transforms.ToTensor()\n    \n    if split:\n        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=split)\n    \n        train_dataset = MNIST(X_train, y_train, transform=transform)\n        test_dataset = MNIST(X_test, y_test, transform=transform)\n        \n        train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n        test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=True)\n\n        return train_dataset, test_dataset, train_loader, test_loader\n    \n    dataset = MNIST(X, y, transform=transform)\n    loader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n    return dataset, loader","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:22:30.930215Z","iopub.execute_input":"2022-11-21T15:22:30.930594Z","iopub.status.idle":"2022-11-21T15:22:30.942918Z","shell.execute_reply.started":"2022-11-21T15:22:30.930561Z","shell.execute_reply":"2022-11-21T15:22:30.942096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data, test_data, train_loader, test_loader = get_loader(train, 0.2)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:22:30.946008Z","iopub.execute_input":"2022-11-21T15:22:30.946642Z","iopub.status.idle":"2022-11-21T15:22:31.655821Z","shell.execute_reply.started":"2022-11-21T15:22:30.946606Z","shell.execute_reply":"2022-11-21T15:22:31.654545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvNet(nn.Module):\n    def __init__(self, size_out, size_emb):\n        super().__init__()\n        \n        self.model = nn.Sequential(\n            nn.Conv2d(1, 64, kernel_size=4, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(64),\n            nn.LeakyReLU(0.2, inplace=True),\n\n            nn.Conv2d(64, 128, kernel_size=4, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(128),\n            nn.LeakyReLU(0.2, inplace=True),\n\n            nn.Conv2d(128, 256, kernel_size=4, stride=2, padding=1, bias=False),\n            nn.BatchNorm2d(256),\n            nn.LeakyReLU(0.2, inplace=True),\n\n            nn.Conv2d(256, size_emb, kernel_size=3, stride=1, padding=0, bias=False),\n            nn.Flatten(),\n            nn.Linear(size_emb, size_out),\n            nn.Softmax()\n        )\n    \n    def forward(self, x):\n        return self.model(x)\n\n\nclass LeNet5(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        self.model = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, stride=1, bias=False),\n            nn.BatchNorm2d(32),\n            nn.LeakyReLU(0.2, inplace=True),\n            nn.MaxPool2d(kernel_size=2,stride=2),\n            nn.Dropout(p=0.2, inplace=True),\n\n            nn.Conv2d(32, 64, kernel_size=3, stride=1, bias=False),\n            nn.BatchNorm2d(64),\n            nn.LeakyReLU(0.2, inplace=True),\n            nn.MaxPool2d(kernel_size=2,stride=2),\n            nn.Dropout(p=0.2, inplace=True),\n\n            nn.Flatten(),\n            nn.Linear(1600, 100),\n            nn.ReLU(inplace=True),\n            nn.Linear(100, 10),\n            nn.Softmax()\n        )\n    \n    def forward(self, x):\n        return self.model(x)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T16:51:01.894704Z","iopub.execute_input":"2022-11-21T16:51:01.895113Z","iopub.status.idle":"2022-11-21T16:51:01.910852Z","shell.execute_reply.started":"2022-11-21T16:51:01.895082Z","shell.execute_reply":"2022-11-21T16:51:01.909712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = ConvNet(10, EMB_SIZE)\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:22:31.676960Z","iopub.execute_input":"2022-11-21T15:22:31.677785Z","iopub.status.idle":"2022-11-21T15:22:31.707848Z","shell.execute_reply.started":"2022-11-21T15:22:31.677721Z","shell.execute_reply":"2022-11-21T15:22:31.706179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_losses = []\ntest_losses = []\ntrain_correct = []\ntest_correct = []\n\nfor i in range(EPOCHS):\n    train_corr = 0\n    test_corr = 0\n\n    for image_data, image_target in tqdm(train_loader):\n        preds = model(image_data.float())\n        loss = criterion(preds, image_target)\n\n        predicted = torch.max(preds.data, 1)[1]\n        batch_corr = (predicted==image_target).sum()\n        train_corr += batch_corr\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    \n    train_losses.append(loss)\n    train_correct.append(train_corr)\n    \n    with torch.no_grad():\n        for image_data, image_target in test_loader:\n            preds = model(image_data.float())\n\n            predicted = torch.max(preds.data, 1)[1] \n            test_corr += (predicted == image_target).sum()\n    \n    test_loss = criterion(preds, image_target)\n    test_losses.append(test_loss)\n    test_correct.append(test_corr)\n    \n    train_acc = train_corr.item()*100/(BATCH_SIZE*len(train_loader))\n    test_acc = test_corr.item()*100/(BATCH_SIZE*len(test_loader))\n\n    print(f'epoch: {i:2}  loss: {loss.item():10.8f}  train_accuracy: {train_acc:7.3f}%  test_accuracy: {test_acc:7.3f}%')","metadata":{"execution":{"iopub.status.busy":"2022-11-21T15:27:28.458948Z","iopub.execute_input":"2022-11-21T15:27:28.460040Z","iopub.status.idle":"2022-11-21T16:49:38.462743Z","shell.execute_reply.started":"2022-11-21T15:27:28.459941Z","shell.execute_reply":"2022-11-21T16:49:38.461347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lenet = LeNet5()\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_lenet.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T16:51:16.815086Z","iopub.execute_input":"2022-11-21T16:51:16.815568Z","iopub.status.idle":"2022-11-21T16:51:16.826477Z","shell.execute_reply.started":"2022-11-21T16:51:16.815528Z","shell.execute_reply":"2022-11-21T16:51:16.825223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le_train_losses = []\nle_test_losses = []\nle_train_acc = []\nle_test_acc = []\n\nfor i in range(EPOCHS):\n    train_corr = 0\n    test_corr = 0\n\n    for image_data, image_target in tqdm(train_loader):\n        preds = model_lenet(image_data.float())\n        loss = criterion(preds, image_target)\n\n        predicted = torch.max(preds.data, 1)[1]\n        batch_corr = (predicted==image_target).sum()\n        train_corr += batch_corr\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    \n    with torch.no_grad():\n        for image_data, image_target in test_loader:\n            preds = model_lenet(image_data.float())\n\n            predicted = torch.max(preds.data, 1)[1] \n            test_corr += (predicted == image_target).sum()\n    \n    test_loss = criterion(preds, image_target)\n    \n    train_acc = train_corr.item()*100/(BATCH_SIZE*len(train_loader))\n    test_acc = test_corr.item()*100/(BATCH_SIZE*len(test_loader))\n    \n    le_train_losses.append(loss)\n    le_train_acc.append(train_acc)\n    le_test_losses.append(test_loss)\n    le_test_acc.append(test_acc)\n\n    print(f'epoch: {i:2}  loss: {loss.item():10.8f}  train_accuracy: {train_acc:7.3f}%  test_accuracy: {test_acc:7.3f}%')","metadata":{"execution":{"iopub.status.busy":"2022-11-21T16:54:05.416442Z","iopub.execute_input":"2022-11-21T16:54:05.417824Z","iopub.status.idle":"2022-11-21T17:15:00.288223Z","shell.execute_reply.started":"2022-11-21T16:54:05.417779Z","shell.execute_reply":"2022-11-21T17:15:00.286982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset, loader = get_loader(train)\nEPOCHS = 100\n\nmodel_lenet = LeNet5()\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_lenet.parameters(), lr=0.001)\n\nfor i in range(EPOCHS):\n    train_corr = 0\n\n    for image_data, image_target in tqdm(loader):\n        preds = model_lenet(image_data.float())\n        loss = criterion(preds, image_target)\n\n        predicted = torch.max(preds.data, 1)[1]\n        batch_corr = (predicted==image_target).sum()\n        train_corr += batch_corr\n        \n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    \n    train_acc = train_corr.item()*100/(BATCH_SIZE*len(loader))\n\n    print(f'epoch: {i:2}  loss: {loss.item():10.8f}  train_accuracy: {train_acc:7.3f}%')","metadata":{"execution":{"iopub.status.busy":"2022-11-21T17:46:37.576356Z","iopub.execute_input":"2022-11-21T17:46:37.577742Z","iopub.status.idle":"2022-11-21T18:37:42.914564Z","shell.execute_reply.started":"2022-11-21T17:46:37.577686Z","shell.execute_reply":"2022-11-21T18:37:42.913323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_lenet.eval()\ntest_data = torch.tensor(test.values.reshape(-1, 1, 28, 28) / 255)\n\nwith torch.no_grad():\n    preds = model_lenet(test_data.float())\n    predicted = torch.max(preds.data, 1)[1]","metadata":{"execution":{"iopub.status.busy":"2022-11-21T19:18:22.114340Z","iopub.execute_input":"2022-11-21T19:18:22.114809Z","iopub.status.idle":"2022-11-21T19:18:34.659604Z","shell.execute_reply.started":"2022-11-21T19:18:22.114772Z","shell.execute_reply":"2022-11-21T19:18:34.658337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'ImageId': list(range(1, len(predicted)+1)), 'Label': predicted})\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-21T19:20:16.237217Z","iopub.execute_input":"2022-11-21T19:20:16.237632Z","iopub.status.idle":"2022-11-21T19:20:16.288968Z","shell.execute_reply.started":"2022-11-21T19:20:16.237592Z","shell.execute_reply":"2022-11-21T19:20:16.288065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}