{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom tqdm.notebook import tqdm\nimport torch\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torch.utils.data import DataLoader\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:43.669534Z","iopub.execute_input":"2022-05-28T16:41:43.669936Z","iopub.status.idle":"2022-05-28T16:41:46.552363Z","shell.execute_reply.started":"2022-05-28T16:41:43.669862Z","shell.execute_reply":"2022-05-28T16:41:46.551565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare Dataset","metadata":{}},{"cell_type":"code","source":"path_to_train_csv = '/kaggle/input/digit-recognizer/train.csv'\ntrain = pd.read_csv(path_to_train_csv, dtype = np.float32)\n\ny = train.label.values\nX = train.loc[:, train.columns != \"label\"].values / 255 # normalization\n\n# Train 90%, Val 10%\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size = 0.1, random_state = 42) \n\n# Convert to Tensor\nX_train = torch.from_numpy(X_train)\ny_train = torch.from_numpy(y_train).type(torch.LongTensor)\nX_val = torch.from_numpy(X_val)\ny_val = torch.from_numpy(y_val).type(torch.LongTensor)\nprint(f'Train images: {len(X_train)}')\nprint(f'Validation images: {len(X_val)}')\n\n# Pytorch train and test sets\ntrain_dataset = torch.utils.data.TensorDataset(X_train, y_train)\nval_dataset = torch.utils.data.TensorDataset(X_val, y_val)\n\n# Data loaders\nbatch_size = 100 # Use multiples of 100\ntrain_loader = DataLoader(train_dataset, batch_size = batch_size, shuffle = False)\nval_loader = DataLoader(val_dataset, batch_size = batch_size, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:46.559443Z","iopub.execute_input":"2022-05-28T16:41:46.560006Z","iopub.status.idle":"2022-05-28T16:41:50.407659Z","shell.execute_reply.started":"2022-05-28T16:41:46.559971Z","shell.execute_reply":"2022-05-28T16:41:50.406811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize one of the images in data set\nindex = 100\nplt.imshow(X_train[index].reshape(28,28))\nplt.axis(\"off\")\nplt.title(str(y_train[index]))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:50.409531Z","iopub.execute_input":"2022-05-28T16:41:50.410323Z","iopub.status.idle":"2022-05-28T16:41:50.521716Z","shell.execute_reply.started":"2022-05-28T16:41:50.410285Z","shell.execute_reply":"2022-05-28T16:41:50.520894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"# Define a CNN model\nclass CNNModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        # Convolution 1\n        self.cnn1 = nn.Conv2d(in_channels = 1, out_channels = 32, kernel_size = (3, 3), stride = 1, padding = 0)\n        self.relu1 = nn.ReLU()\n        self.bn1 = nn.BatchNorm2d(32)\n        self.maxpool1 = nn.MaxPool2d(kernel_size = (2, 2))\n     \n        # Convolution 2\n        self.cnn2 = nn.Conv2d(in_channels = 32, out_channels = 32, kernel_size= (3, 3), stride = 1, padding = 0)\n        self.relu2 = nn.ReLU()\n        self.bn2 = nn.BatchNorm2d(32)\n        self.maxpool2 = nn.MaxPool2d(kernel_size = (2, 2))\n\n        self.fc1 = nn.Linear(32 * 5 * 5, 512) # C * H * W\n        self.fc2 = nn.Linear(512, 10)\n    \n    def forward(self, x):\n        # Convolution 1\n        out = self.cnn1(x)\n        out = self.relu1(out)\n        out = self.bn1(out)\n        out = self.maxpool1(out)\n       \n        # Convolution 2 \n        out = self.cnn2(out)\n        out = self.relu2(out)\n        out = self.bn2(out)\n        out = self.maxpool2(out)\n        \n        # Final layer\n        out = out.view(out.size(0), -1)\n        out = self.fc1(out)\n        \n        return out","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:50.526109Z","iopub.execute_input":"2022-05-28T16:41:50.528639Z","iopub.status.idle":"2022-05-28T16:41:50.545482Z","shell.execute_reply.started":"2022-05-28T16:41:50.528586Z","shell.execute_reply":"2022-05-28T16:41:50.544401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create model\nmodel = CNNModel()\n\n# Cross Entropy Loss \nloss_fn = nn.CrossEntropyLoss()\n\n# Hyperparams\nnum_epochs = 100\n\n# Adam Optimizer\nlearning_rate = 0.001\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:50.546952Z","iopub.execute_input":"2022-05-28T16:41:50.547606Z","iopub.status.idle":"2022-05-28T16:41:50.573337Z","shell.execute_reply.started":"2022-05-28T16:41:50.547567Z","shell.execute_reply":"2022-05-28T16:41:50.572448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RunningAverage:\n    '''\n    Class to compute running average\n    '''\n    def __init__(self):\n        self.count = 0\n        self.total = 0\n    \n    def update(self, value):\n        self.total += value\n        self.count += 1\n    \n    def calculate(self):\n        try: \n            return self.total / self.count\n        except:\n            return 0","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:50.574469Z","iopub.execute_input":"2022-05-28T16:41:50.574943Z","iopub.status.idle":"2022-05-28T16:41:50.581932Z","shell.execute_reply.started":"2022-05-28T16:41:50.574908Z","shell.execute_reply":"2022-05-28T16:41:50.581179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"losses = []\naccuracies = []\nbest_val_loss = 1e7\nbest_state_dict = None\n\n# For each epoch, train and evaluate the val loss and val accuracy\nfor epoch in range(1, num_epochs + 1):\n    print(f'Epoch {epoch}')\n    train_bar = tqdm(range(len(train_loader)), desc = 'Training', leave = False)\n\n    model.train()\n    for i, (feature, labels) in enumerate(train_loader):\n\n        image = Variable(feature.view(batch_size, 1, 28, 28))\n        labels = Variable(labels)\n\n        # Clear gradients\n        optimizer.zero_grad()\n\n        # Forward propagation\n        outputs = model(image)\n\n        # Calculate softmax and ross entropy loss\n        loss = loss_fn(outputs, labels)\n\n        # Calculating gradients\n        loss.backward()\n\n        # Update parameters\n        optimizer.step()\n        \n        train_bar.update()\n    \n    val_bar = tqdm(range(len(val_loader)), desc = 'Evaluation', leave = False)\n    running_loss = RunningAverage()\n    running_accuracy = RunningAverage()\n    model.eval()\n    for i, (feature, labels) in enumerate(val_loader):\n        image = Variable(feature.view(batch_size, 1, 28, 28))\n        labels = Variable(labels)\n        with torch.no_grad():\n            # Forward propagation\n            outputs = model(image)\n\n            # Calculate crosee entropy loss\n            batch_loss = loss_fn(outputs, labels)\n            batch_accuracy = torch.mean((torch.argmax(outputs, dim = 1) == labels).to(float))\n\n            # Calculate loss and accuracy\n            running_loss.update(batch_loss)\n            running_accuracy.update(batch_accuracy)\n        \n        val_bar.update()\n\n    loss, accuracy = running_loss.calculate(), running_accuracy.calculate()\n    \n    if best_val_loss > loss:\n        best_state_dict = model.state_dict()\n    \n    losses.append(loss)\n    accuracies.append(accuracy)\n    val_bar.set_postfix_str(f'Loss {loss:.4f}-Accuracy {accuracy:.4f}')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T16:41:50.583745Z","iopub.execute_input":"2022-05-28T16:41:50.584009Z","iopub.status.idle":"2022-05-28T17:23:51.277785Z","shell.execute_reply.started":"2022-05-28T16:41:50.583985Z","shell.execute_reply":"2022-05-28T17:23:51.276846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot losses\nplt.plot([i for i in range(len(losses))], losses)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:23:51.279253Z","iopub.execute_input":"2022-05-28T17:23:51.27963Z","iopub.status.idle":"2022-05-28T17:23:51.45696Z","shell.execute_reply.started":"2022-05-28T17:23:51.279593Z","shell.execute_reply":"2022-05-28T17:23:51.4562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot accuracies\nplt.plot([i for i in range(len(accuracies))], accuracies)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:23:51.627619Z","iopub.execute_input":"2022-05-28T17:23:51.62864Z","iopub.status.idle":"2022-05-28T17:23:51.79942Z","shell.execute_reply.started":"2022-05-28T17:23:51.6286Z","shell.execute_reply":"2022-05-28T17:23:51.798641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate prediction","metadata":{}},{"cell_type":"code","source":"# Load best state dict\nmodel = CNNModel()\nmodel.load_state_dict(best_state_dict)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:24:01.470883Z","iopub.execute_input":"2022-05-28T17:24:01.471229Z","iopub.status.idle":"2022-05-28T17:24:01.485153Z","shell.execute_reply.started":"2022-05-28T17:24:01.471199Z","shell.execute_reply":"2022-05-28T17:24:01.484051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_test_csv = '/kaggle/input/digit-recognizer/test.csv'\ntest = pd.read_csv(path_to_test_csv, dtype = np.float32).values / 255\nprint(f'Test size : {len(test)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:31:49.402711Z","iopub.execute_input":"2022-05-28T17:31:49.403058Z","iopub.status.idle":"2022-05-28T17:31:51.32596Z","shell.execute_reply.started":"2022-05-28T17:31:49.40303Z","shell.execute_reply":"2022-05-28T17:31:51.325063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = torch.from_numpy(test)\ntest_dataset = torch.utils.data.TensorDataset(X_test)\n\n# Data loaders\nbatch_size = 100 # Use multiples of 100\ntest_loader = DataLoader(test_dataset, batch_size = batch_size, shuffle = False)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:33:29.772803Z","iopub.execute_input":"2022-05-28T17:33:29.773179Z","iopub.status.idle":"2022-05-28T17:33:29.777918Z","shell.execute_reply.started":"2022-05-28T17:33:29.773149Z","shell.execute_reply":"2022-05-28T17:33:29.777157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Start prediction\nans = []\n\nmodel.eval()\nfor feature in test_loader:\n    image = Variable(feature[0].view(batch_size, 1, 28, 28))\n    with torch.no_grad():\n        # Forward propagation\n        outputs = model(image)\n\n        # Calculate crosee entropy loss\n        predictions = torch.argmax(outputs, dim = 1)\n\n        ans.extend(predictions)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:33:30.000225Z","iopub.execute_input":"2022-05-28T17:33:30.000629Z","iopub.status.idle":"2022-05-28T17:33:35.173797Z","shell.execute_reply.started":"2022-05-28T17:33:30.000598Z","shell.execute_reply":"2022-05-28T17:33:35.172918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path_to_sample = '/kaggle/input/digit-recognizer/sample_submission.csv'\ndf = pd.read_csv(path_to_sample)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:35:23.818172Z","iopub.execute_input":"2022-05-28T17:35:23.818536Z","iopub.status.idle":"2022-05-28T17:35:23.843002Z","shell.execute_reply.started":"2022-05-28T17:35:23.818508Z","shell.execute_reply":"2022-05-28T17:35:23.842277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['Label'] = [item.item() for item in ans]","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:35:47.002728Z","iopub.execute_input":"2022-05-28T17:35:47.003069Z","iopub.status.idle":"2022-05-28T17:35:47.033584Z","shell.execute_reply.started":"2022-05-28T17:35:47.003041Z","shell.execute_reply":"2022-05-28T17:35:47.032824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('submission1.csv', index = None)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T17:36:00.030808Z","iopub.execute_input":"2022-05-28T17:36:00.031147Z","iopub.status.idle":"2022-05-28T17:36:00.076762Z","shell.execute_reply.started":"2022-05-28T17:36:00.031111Z","shell.execute_reply":"2022-05-28T17:36:00.076021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}