{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer - PyTorch\n\nhttps://www.kaggle.com/competitions/digit-recognizer","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"}},{"cell_type":"markdown","source":"## Setup","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch import optim\nfrom torch.autograd import Variable\n\nimport torchinfo","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IS_KAGGLE = os.environ.get('KAGGLE_KERNEL_RUN_TYPE', '')\nCOMP_NAME = 'digit-recognizer'\n\nif COMP_NAME is None:\n    raise NameError('COMP_NAME has not been initialized')\n\nRANDOM_SEED = 42\n\nBATCH_SIZE = 100\n\nDATA_PATH = Path('../input/' + COMP_NAME) if IS_KAGGLE else Path('./data')\n\nDEVICE = 'cuda:0' if torch.cuda.is_available() else 'cpu'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('kaggle:', 'Y' if IS_KAGGLE else 'N')\nprint('torch version:', torch.__version__)\nprint('device:', DEVICE)\nprint(torch.cuda.device_count(), 'GPU(s) available')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reproducibility\n\nrandom.seed(RANDOM_SEED)\nnp.random.seed(RANDOM_SEED)\ntorch.manual_seed(RANDOM_SEED)\ntorch.cuda.manual_seed_all(RANDOM_SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data\n\nhttps://www.kaggle.com/competitions/digit-recognizer/data","metadata":{}},{"cell_type":"code","source":"path = Path('./data')\nif not DATA_PATH.exists():\n    import zipfile, kaggle\n    kaggle.api.competition_download_cli(COMP_NAME)\n    zipfile.ZipFile(f'{COMP_NAME}.zip').extractall(DATA_PATH)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(DATA_PATH / 'train.csv')\ntest_data = pd.read_csv(DATA_PATH / 'test.csv')\n\nprint('train:', train_data.shape)\nprint('test: ', test_data.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_data.iloc[:, train_data.columns != 'label']\ny = train_data.label.values\n\nprint('X:', X.shape, type(X))\nprint('y:', y.shape, type(y))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Exploration","metadata":{}},{"cell_type":"code","source":"figure = plt.figure(figsize=(8,8))\ncols, rows = 3, 3\n\nfor i in range(1, cols * rows + 1):\n    sample = random.randint(0, len(X)-1)\n    label = str(int(y[sample]))\n    img = X.values[sample].reshape(28,28)\n    \n    figure.add_subplot(rows, cols, i)\n    plt.title(label)\n    plt.axis('off')\n    plt.imshow(img, cmap='gray')\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['label'].value_counts()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare Training/Validation Data","metadata":{}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2, random_state=RANDOM_SEED)\n\nprint('X_train:', X_train.shape, 'y_train:', y_train.shape)\nprint('X_valid:', X_valid.shape, 'y_valid:', y_valid.shape)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# normalize (0-255 -> 0-1)\n\nX_train = X_train.values / 255\nX_valid = X_valid.values / 255","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = torch.from_numpy(X_train)\ny_train = torch.from_numpy(y_train).type(torch.LongTensor)\n\nX_valid = torch.from_numpy(X_valid)\ny_valid = torch.from_numpy(y_valid).type(torch.LongTensor)\n\nprint('X_train.dtype:', X_train.dtype)\nprint('y_train.dtype:', y_train.dtype)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = torch.utils.data.TensorDataset(X_train, y_train)\nvalid_ds = torch.utils.data.TensorDataset(X_valid, y_valid)\n\ntrain_loader = torch.utils.data.DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True)\nvalid_loader = torch.utils.data.DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build Model","metadata":{}},{"cell_type":"code","source":"class DigitRecognizerModel(nn.Module):\n    \"\"\"\n    Architecture summary:\n      - Layer 1: Convolution 1 > Activation (ReLU) > Pooling 1 > Dropout 1\n      - Layer 2: Convolution 2 > Activation (ReLU) > Pooling 2 > Dropout 2 > Flatten\n      - Layer 3: Linear 1 > Activation (ReLU) > Dropout 3\n      - Layer 4: Linear 2 > Activation (ReLU) > Dropout 4\n      - Layer 5: Output > Activation (Softmax)\n    \"\"\"\n    \n    def __init__(self):\n        super().__init__()\n\n        # layer 1 \n        # (black and white image, 1 color channel == 1 input channel)\n        self.conv1 = nn.Conv2d(in_channels=1, out_channels=128, kernel_size=5, stride=1, padding=0)\n        self.pool1 = nn.MaxPool2d(kernel_size=(2, 2))\n        self.drop1 = nn.Dropout(p=0.3)\n\n        # layer 2\n        self.conv2 = nn.Conv2d(in_channels=128, out_channels=224, kernel_size=5, stride=1, padding=0)\n        self.pool2 = nn.MaxPool2d(kernel_size=(2, 2))\n        self.drop2 = nn.Dropout(p=0.4)\n\n        # layer 3\n        self.linear1 = nn.Linear(in_features=224 * 4 * 4, out_features=64)\n        self.drop3 = nn.Dropout(p=0.4)\n\n        # layer 4\n        self.linear2 = nn.Linear(in_features=64, out_features=32)\n        self.drop4 = nn.Dropout(p=0.4)\n\n        # layer 5\n        self.linear3 = nn.Linear(in_features=32, out_features=10)\n        # self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, x):\n        out = self.drop1(self.pool1(F.relu(self.conv1(x)))) # layer 1\n        out = self.drop2(self.pool2(F.relu(self.conv2(out)))) # layer 2\n        out = out.view(-1, 224 * 4 * 4) # flatten layer 2 outputs since layer 3 is linear\n        \n        out = self.drop3(F.relu(self.linear1(out))) # layer 3\n        out = self.drop4(F.relu(self.linear2(out))) # layer 4\n        # out = self.softmax(self.linear3(out)) # layer 5\n        out = self.linear3(out) # logits\n        return out","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DigitRecognizerModel().to(DEVICE)\n\nloss_fn = nn.CrossEntropyLoss()\nlearning_rate = 0.001\noptimizer = optim.Adam(model.parameters(), lr=learning_rate)\n\nprint(model)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torchinfo.summary(model, (1, 28, 28), col_names=('input_size', 'output_size', 'num_params', 'kernel_size'), verbose=0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Model","metadata":{}},{"cell_type":"code","source":"def train_loop(dataloader, model, loss_fn, optimizer, print_every=100):\n    size = len(dataloader.dataset)\n    model.train()\n    \n    for batch, (X, y) in enumerate(dataloader):\n        X = (X.view(-1, 1, 28, 28)).type(torch.FloatTensor).to(DEVICE)\n        y = y.to(DEVICE)\n\n        logits = model(X)\n        loss = loss_fn(logits, y)\n\n        # backpropagation\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n\n        # print results\n        if batch % print_every == 0:\n            loss, current = loss.item(), (batch + 1) * len(X)\n            print(f'loss: {loss:>7f} [{current:>5d}/{size:>5d}]')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_loop(dataloader, model, loss_fn, print_every=100):\n    size = len(dataloader.dataset)\n    model.eval()\n    batches = len(dataloader)\n    loss, correct = 0, 0\n\n    with torch.no_grad():\n        for X, y in dataloader:\n            X = (X.view(-1, 1, 28, 28)).type(torch.FloatTensor).to(DEVICE)\n            y = y.to(DEVICE)\n\n            logits = model(X)\n            loss += loss_fn(logits, y).item()\n            correct += (logits.argmax(1) == y).type(torch.float).sum().item()\n\n    loss /= BATCH_SIZE\n    correct /= size\n    print(f'Validation: Accuracy={(100*correct):>0.1f}%, Average_Loss={loss:>8f}\\n')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 10\n\nfor epoch in range(epochs):\n    print(f'Epoch {epoch+1}\\n-------------------------------')\n    train_loop(train_loader, model, loss_fn, optimizer)\n    valid_loop(valid_loader, model, loss_fn)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"X_test = test_data.values\nX_test = torch.from_numpy(X_test / 255)\n\ny_test = np.zeros(X_test.shape)\ny_test = torch.from_numpy(y_test)\n\ntest_ds = torch.utils.data.TensorDataset(X_test, y_test)\ntest_loader = torch.utils.data.DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_rows = [['ImageId', 'Label']]\n\nwith torch.no_grad():\n    model.eval()\n    image_id = 1\n\n    for X, _ in test_loader:\n        X = (X.view(-1, 1, 28, 28)).type(torch.FloatTensor).to(DEVICE)\n        preds = model(X).argmax(1)\n\n        for pred in preds:\n            submission_rows.append([image_id, pred.item()])\n            image_id += 1\n\nsubmission = pd.DataFrame(submission_rows)\nsubmission.columns = submission.iloc[0]\nsubmission = submission.drop(0, axis=0)\n\nsubmission.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]}]}