{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30176,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install torchsummary","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-10T17:02:00.200452Z","iopub.execute_input":"2022-04-10T17:02:00.200805Z","iopub.status.idle":"2022-04-10T17:02:11.250813Z","shell.execute_reply.started":"2022-04-10T17:02:00.200771Z","shell.execute_reply":"2022-04-10T17:02:11.250009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport time\nimport os\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom PIL import Image\nimport cv2\nfrom tqdm import tqdm\n\nimport torch \nimport torchvision\nimport torch.nn as nn \nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torchsummary import summary\nfrom torch.utils.data import random_split, DataLoader, TensorDataset\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:02:11.25241Z","iopub.execute_input":"2022-04-10T17:02:11.252699Z","iopub.status.idle":"2022-04-10T17:02:14.211407Z","shell.execute_reply.started":"2022-04-10T17:02:11.252668Z","shell.execute_reply":"2022-04-10T17:02:14.21028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fix the package versions so that the experiments are reproducible\n!pip freeze > requirements.txt","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:02:14.213288Z","iopub.execute_input":"2022-04-10T17:02:14.214787Z","iopub.status.idle":"2022-04-10T17:02:17.733809Z","shell.execute_reply.started":"2022-04-10T17:02:14.214729Z","shell.execute_reply":"2022-04-10T17:02:17.732491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Data load\ntrain_data=pd.read_csv('/kaggle/input/digit-recognizer/train.csv', header='infer')\ntest_data=pd.read_csv('/kaggle/input/digit-recognizer/test.csv', header='infer')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:30.743701Z","iopub.execute_input":"2022-04-10T17:10:30.744216Z","iopub.status.idle":"2022-04-10T17:10:35.328686Z","shell.execute_reply.started":"2022-04-10T17:10:30.744176Z","shell.execute_reply":"2022-04-10T17:10:35.327754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.330643Z","iopub.execute_input":"2022-04-10T17:10:35.330877Z","iopub.status.idle":"2022-04-10T17:10:35.350894Z","shell.execute_reply.started":"2022-04-10T17:10:35.330849Z","shell.execute_reply":"2022-04-10T17:10:35.349898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.sample(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.352121Z","iopub.execute_input":"2022-04-10T17:10:35.352374Z","iopub.status.idle":"2022-04-10T17:10:35.377029Z","shell.execute_reply.started":"2022-04-10T17:10:35.352343Z","shell.execute_reply":"2022-04-10T17:10:35.376035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"x_train = train_data.drop('label', axis=1).values\ny_train = train_data['label'].values\n\nx_test = test_data.values","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.379705Z","iopub.execute_input":"2022-04-10T17:10:35.38034Z","iopub.status.idle":"2022-04-10T17:10:35.468431Z","shell.execute_reply.started":"2022-04-10T17:10:35.380291Z","shell.execute_reply":"2022-04-10T17:10:35.467579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = torch.tensor(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.469905Z","iopub.execute_input":"2022-04-10T17:10:35.470228Z","iopub.status.idle":"2022-04-10T17:10:35.473231Z","shell.execute_reply.started":"2022-04-10T17:10:35.4702Z","shell.execute_reply":"2022-04-10T17:10:35.47266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = transforms.ToTensor()(x_train)\nx_test = transforms.ToTensor()(x_test)\n\nprint(x_train.shape, x_test.shape, sep='\\n')","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.474077Z","iopub.execute_input":"2022-04-10T17:10:35.474535Z","iopub.status.idle":"2022-04-10T17:10:35.711302Z","shell.execute_reply.started":"2022-04-10T17:10:35.474479Z","shell.execute_reply":"2022-04-10T17:10:35.71027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train.reshape(x_train.shape[1],28,28)\n\nx_test = x_test.reshape(x_test.shape[1],28*28)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.714313Z","iopub.execute_input":"2022-04-10T17:10:35.716398Z","iopub.status.idle":"2022-04-10T17:10:35.721165Z","shell.execute_reply.started":"2022-04-10T17:10:35.71636Z","shell.execute_reply":"2022-04-10T17:10:35.720365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train/255\nx_test = x_test/255","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.722554Z","iopub.execute_input":"2022-04-10T17:10:35.722842Z","iopub.status.idle":"2022-04-10T17:10:35.960878Z","shell.execute_reply.started":"2022-04-10T17:10:35.722812Z","shell.execute_reply":"2022-04-10T17:10:35.959975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = torch.tensor(y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.962122Z","iopub.execute_input":"2022-04-10T17:10:35.962364Z","iopub.status.idle":"2022-04-10T17:10:35.96725Z","shell.execute_reply.started":"2022-04-10T17:10:35.962335Z","shell.execute_reply":"2022-04-10T17:10:35.966455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_tensor = TensorDataset(x_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.969865Z","iopub.execute_input":"2022-04-10T17:10:35.970092Z","iopub.status.idle":"2022-04-10T17:10:35.978698Z","shell.execute_reply.started":"2022-04-10T17:10:35.970063Z","shell.execute_reply":"2022-04-10T17:10:35.978018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TR_SIZE = 32000\nVAL_SIZE = 10000\nBATCH_SIZE = 100","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.980057Z","iopub.execute_input":"2022-04-10T17:10:35.980512Z","iopub.status.idle":"2022-04-10T17:10:35.991774Z","shell.execute_reply.started":"2022-04-10T17:10:35.980482Z","shell.execute_reply":"2022-04-10T17:10:35.990867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset, valset = random_split(train_tensor, [TR_SIZE, VAL_SIZE])\n\ntrain_loader = DataLoader(trainset, \n                          batch_size=BATCH_SIZE, \n                          shuffle=True,\n                          num_workers=4)\nval_loader = DataLoader(valset, \n                        batch_size=BATCH_SIZE, \n                        shuffle=True,\n                        num_workers=4)\ntest_loader = DataLoader(x_test, \n                         batch_size=BATCH_SIZE, \n                         shuffle=False,\n                         num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:35.993109Z","iopub.execute_input":"2022-04-10T17:10:35.993626Z","iopub.status.idle":"2022-04-10T17:10:36.018975Z","shell.execute_reply.started":"2022-04-10T17:10:35.993592Z","shell.execute_reply":"2022-04-10T17:10:36.017753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vizualization","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\nk=0\nfor i in range(5):\n    for j in range(5):\n        plt.subplot(5,5,k+1);\n        plt.imshow(x_train[k]);\n        k+=1;","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:36.020541Z","iopub.execute_input":"2022-04-10T17:10:36.020805Z","iopub.status.idle":"2022-04-10T17:10:38.542589Z","shell.execute_reply.started":"2022-04-10T17:10:36.020774Z","shell.execute_reply":"2022-04-10T17:10:38.541781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (15, 10))\nax = sns.countplot(x = 'label', data = train_data)\nplt.title('Class Count', fontsize = 20)\nplt.xlabel('Labels', fontsize = 15)\nplt.ylabel('Count', fontsize = 15)\nfor p in ax.patches:\n        ax.annotate('{:.0f}'.format(p.get_height()), (p.get_x() + 0.25, p.get_height() + 10))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.543948Z","iopub.execute_input":"2022-04-10T17:10:38.544179Z","iopub.status.idle":"2022-04-10T17:10:38.838435Z","shell.execute_reply.started":"2022-04-10T17:10:38.544149Z","shell.execute_reply":"2022-04-10T17:10:38.837834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.839696Z","iopub.execute_input":"2022-04-10T17:10:38.840096Z","iopub.status.idle":"2022-04-10T17:10:38.845713Z","shell.execute_reply.started":"2022-04-10T17:10:38.840066Z","shell.execute_reply":"2022-04-10T17:10:38.844794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputSize = 784\nnumClasses = 10\nnumEpochs = 25\nlearningRate = 0.1","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.847174Z","iopub.execute_input":"2022-04-10T17:10:38.847713Z","iopub.status.idle":"2022-04-10T17:10:38.855862Z","shell.execute_reply.started":"2022-04-10T17:10:38.847662Z","shell.execute_reply":"2022-04-10T17:10:38.85497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.conv_block = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(32),\n            nn.GELU(),\n            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(64),\n            nn.GELU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(128),\n            nn.GELU(),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Conv2d(128, 256, kernel_size=3, stride=1, padding = 1),\n            nn.BatchNorm2d(256),\n            nn.GELU(),\n            nn.MaxPool2d(2)\n        )\n        self.linear_block = nn.Sequential(\n            nn.Dropout(p=0.5),\n            nn.Linear(256*3*3, 256),\n            nn.BatchNorm1d(256),\n            nn.GELU(),\n            nn.Dropout(0.5),\n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n            nn.GELU(),\n            nn.Dropout(0.5),\n            nn.Linear(128, 32),\n            nn.BatchNorm1d(32),\n            nn.GELU(),\n            nn.Dropout(0.5),\n            nn.Linear(32, 10)\n        )\n        \n    def forward(self, x):\n        x = self.conv_block(x)\n        x = x.view(x.size(0), -1)\n        x = self.linear_block(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.85693Z","iopub.execute_input":"2022-04-10T17:10:38.85713Z","iopub.status.idle":"2022-04-10T17:10:38.870951Z","shell.execute_reply.started":"2022-04-10T17:10:38.857105Z","shell.execute_reply":"2022-04-10T17:10:38.869866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN().to(device)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.872114Z","iopub.execute_input":"2022-04-10T17:10:38.872341Z","iopub.status.idle":"2022-04-10T17:10:38.898911Z","shell.execute_reply.started":"2022-04-10T17:10:38.872313Z","shell.execute_reply":"2022-04-10T17:10:38.897991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(model, (1,28,28))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.900131Z","iopub.execute_input":"2022-04-10T17:10:38.900356Z","iopub.status.idle":"2022-04-10T17:10:38.939913Z","shell.execute_reply.started":"2022-04-10T17:10:38.900328Z","shell.execute_reply":"2022-04-10T17:10:38.937926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss() # loss-function\noptimizer = torch.optim.SGD(model.parameters(), lr=learningRate)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.943331Z","iopub.execute_input":"2022-04-10T17:10:38.943623Z","iopub.status.idle":"2022-04-10T17:10:38.948554Z","shell.execute_reply.started":"2022-04-10T17:10:38.943585Z","shell.execute_reply":"2022-04-10T17:10:38.947919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train & val\ndef train_model(num_epoch):\n    model.train()\n    for batch_idx, (images, labels) in enumerate(train_loader):\n        images = images.unsqueeze(1).to(device)\n        labels = labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        if (batch_idx + 1)% 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                  num_epoch, (batch_idx + 1) * len(images), len(train_loader.dataset),\n                  100. * (batch_idx + 1) / len(train_loader), loss.data))\n            \ndef val_model(data_loader):\n    model.eval()\n    loss = 0\n    correct = 0\n    for images, labels in data_loader:\n        images = images.unsqueeze(1).to(device)\n        labels = labels.to(device)\n        outputs = model(images)\n        loss += F.cross_entropy(outputs, labels, size_average=False).data\n        pred = outputs.data.max(1, keepdim=True)[1]\n        correct += pred.eq(labels.data.view_as(pred)).cpu().sum()\n    loss /= len(data_loader.dataset)\n    print('\\nAverage Val Loss: {:.4f}, Val Accuracy: {}/{} ({:.4f}%)\\n'.format(\n        loss, correct, len(data_loader.dataset),\n        100. * correct / len(data_loader.dataset)))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.949767Z","iopub.execute_input":"2022-04-10T17:10:38.950125Z","iopub.status.idle":"2022-04-10T17:10:38.96167Z","shell.execute_reply.started":"2022-04-10T17:10:38.950088Z","shell.execute_reply":"2022-04-10T17:10:38.960943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n in range(numEpochs):\n    train_model(n)\n    val_model(val_loader)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:38.962788Z","iopub.execute_input":"2022-04-10T17:10:38.963513Z","iopub.status.idle":"2022-04-10T17:18:11.643095Z","shell.execute_reply.started":"2022-04-10T17:10:38.963474Z","shell.execute_reply":"2022-04-10T17:18:11.641983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing","metadata":{}},{"cell_type":"code","source":"# test\nmodel.eval()\nlist_reply = []\nwith torch.no_grad():\n    for images in test_loader:\n        images = images.view(-1, 1, 28, 28).to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs.data, 1)\n        for y in predicted:\n            list_reply.append(int(y))","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:19:06.647248Z","iopub.execute_input":"2022-04-10T17:19:06.647662Z","iopub.status.idle":"2022-04-10T17:19:44.388072Z","shell.execute_reply.started":"2022-04-10T17:19:06.647621Z","shell.execute_reply":"2022-04-10T17:19:44.386945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:09:58.346809Z","iopub.execute_input":"2022-04-10T17:09:58.347229Z","iopub.status.idle":"2022-04-10T17:09:58.376974Z","shell.execute_reply.started":"2022-04-10T17:09:58.347182Z","shell.execute_reply":"2022-04-10T17:09:58.375942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:09:58.632974Z","iopub.execute_input":"2022-04-10T17:09:58.633611Z","iopub.status.idle":"2022-04-10T17:09:58.646143Z","shell.execute_reply.started":"2022-04-10T17:09:58.633547Z","shell.execute_reply":"2022-04-10T17:09:58.644985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['Label'] = list_reply\n\nsubmission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:09:58.977236Z","iopub.execute_input":"2022-04-10T17:09:58.977597Z","iopub.status.idle":"2022-04-10T17:09:59.049885Z","shell.execute_reply.started":"2022-04-10T17:09:58.977562Z","shell.execute_reply":"2022-04-10T17:09:59.048718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-04-10T17:10:02.936945Z","iopub.execute_input":"2022-04-10T17:10:02.937838Z","iopub.status.idle":"2022-04-10T17:10:02.949625Z","shell.execute_reply.started":"2022-04-10T17:10:02.937782Z","shell.execute_reply":"2022-04-10T17:10:02.948635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}