{"cells":[{"metadata":{},"cell_type":"markdown","source":"# I can't overfit...\nIn this kernel I train very wide neural network without dropout and other regularization.  \nIt still works ok and it barely overfits."},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# Import Time\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nimport torchvision\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom tqdm import tqdm_notebook as tqdm\n\n%matplotlib inline\n\nBATCH_SIZE = 256\n\nprint(os.listdir('../input/Kannada-MNIST/'))\n\n# Gets the GPU if there is one, otherwise the cpu\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntarget = df_train['label']\ndf_train.drop('label', axis=1, inplace=True)\n\nX_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\nX_test.drop('id', axis=1, inplace=True)\n\nX_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_dev, y_train, y_dev = train_test_split(df_train, target, stratify=target, random_state=42, test_size=0.2)\nprint('X_train', len(X_train))\nprint('X_dev', len(X_dev))\nprint('X_test', len(X_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualization Reference Kernel https://www.kaggle.com/josephvm/kannada-with-pytorch\n# Some quick data visualization \n# First 10 images of each class in the training set\n\nfig, ax = plt.subplots(nrows=10, ncols=10, figsize=(10,10))\n\n# I know these for loops look weird, but this way num_i is only computed once for each class\nfor i in range(10): # Column by column\n    num_i = X_train[y_train == i]\n    ax[0][i].set_title(i)\n    for j in range(10): # Row by row\n        ax[j][i].axis('off')\n        ax[j][i].imshow(num_i.iloc[j, :].to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# PyTorch data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CharData(Dataset):\n    \"\"\"\n    Infer from standard PyTorch Dataset class\n    Such datasets are often very useful\n    \"\"\"\n    \n    def __init__(self,\n                 images,\n                 labels=None,\n                 transform=None,\n                ):\n        self.X = images\n        self.y = labels\n        \n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        img = np.array(self.X.iloc[idx, :], dtype='uint8').reshape([28, 28, 1])\n        if self.transform is not None:\n            img = self.transform(img)\n        \n        if self.y is not None:\n            y = np.zeros(10, dtype='float32')\n            y[self.y.iloc[idx]] = 1\n            return img, y\n        else:\n            return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Put some augmentation on training data\ntrain_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomAffine(degrees=5, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=5),\n    transforms.ToTensor()\n])\n\n# Test data without augmentation\ntest_transform = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.ToTensor()\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create dataset objects\ntrain_dataset = CharData(X_train, y_train, train_transform)\ndev_dataset = CharData(X_dev, y_dev, test_transform)\ntest_dataset = CharData(X_test, transform=test_transform)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create data generators - they will produce batches\ntrain_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)\ndev_loader = DataLoader(dataset=dev_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Show some generated samples"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=16, figsize=(30,4))\n\nfor batch in train_loader:\n    for i in range(16):\n        ax[i].set_title(batch[1][i].data.numpy().argmax())\n        ax[i].imshow(batch[0][i, 0], cmap='gray')\n    break","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Define model"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEPTH_MULT = 4\n\nclass ConvLayer(nn.Module):\n    def __init__(self, input_size, output_size, kernel_size=3):\n        super(ConvLayer, self).__init__()\n        self.ops = nn.Sequential(\n            nn.Conv2d(input_size, output_size, kernel_size=kernel_size, stride=1, padding=kernel_size//2),\n            nn.BatchNorm2d(output_size),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        return self.ops(x)\n\n\nclass FCLayer(nn.Module):\n    def __init__(self, input_size, output_size):\n        super(FCLayer, self).__init__()\n        self.ops = nn.Sequential(\n            nn.Linear(input_size, output_size),\n            nn.BatchNorm1d(output_size),\n            nn.ReLU(inplace=True)\n        )\n    \n    def forward(self, x):\n        return self.ops(x)\n\n\ndef mixup(x, shuffle, lam, i, j):\n    if shuffle is not None and lam is not None and i == j:\n        x = lam * x + (1 - lam) * x[shuffle]\n    return x\n\n\nclass Net(nn.Module):\n    def __init__(self, num_classes):\n        super(Net, self).__init__()\n        \n        self.conv1 = ConvLayer(1, DEPTH_MULT * 32)\n        self.conv2 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        self.conv3 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        self.conv4 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 32)\n        \n        self.conv5 = ConvLayer(DEPTH_MULT * 32, DEPTH_MULT * 64)\n        self.conv6 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv7 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv8 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv9 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        self.conv10 = ConvLayer(DEPTH_MULT * 64, DEPTH_MULT * 64)\n        \n        self.mp = nn.MaxPool2d(kernel_size=2, stride=2)\n        \n        self.fc1 = FCLayer(DEPTH_MULT * 64 * 7 * 7, DEPTH_MULT * 512)\n        self.fc2 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.fc3 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.fc4 = FCLayer(DEPTH_MULT * 512, DEPTH_MULT * 512)\n        self.projection = nn.Linear(DEPTH_MULT * 512, 10)\n    \n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        x = self.conv3(x)\n        x = self.conv4(x)\n        x = self.mp(x)\n        \n        x = self.conv5(x)\n        x = self.conv6(x)\n        x = self.conv7(x)\n        x = self.conv8(x)\n        x = self.conv9(x)\n        x = self.conv10(x)\n        x = self.mp(x)\n        \n        x = x.view(x.size(0), -1)\n        x = self.fc1(x)\n        x = self.fc2(x)\n        x = self.fc3(x)\n        x = self.fc4(x)\n        x = self.projection(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def criterion(input, target, size_average=True):\n    \"\"\"Categorical cross-entropy with logits input and one-hot target\"\"\"\n    l = -(target * torch.log(F.softmax(input, dim=1) + 1e-10)).sum(1)\n    if size_average:\n        l = l.mean()\n    else:\n        l = l.sum()\n    return l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Net(10)\nmodel = model.to(device)\n\nn_epochs = 80\n\noptimizer = optim.Adam(model.parameters(), lr=0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=n_epochs // 4, gamma=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(epoch, history=None):\n    model.train()\n\n    for batch_idx, (data, target) in enumerate(train_loader):\n        data = data.to(device)\n        target = target.to(device)\n                \n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        if history is not None:\n            history.loc[epoch + batch_idx / len(train_loader), 'train_loss'] = loss.data.cpu().numpy()\n        \n        loss.backward()\n        optimizer.step()\n        \n        if (batch_idx + 1) % 100 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLR: {:.6f}\\tLoss: {:.6f}'.format(\n                epoch, (batch_idx + 1) * len(data), len(train_loader.dataset),\n                100. * (batch_idx + 1) / len(train_loader),\n                optimizer.state_dict()['param_groups'][0]['lr'],\n                loss.data))\n    exp_lr_scheduler.step()\n\ndef evaluate(epoch, history=None):\n    model.eval()\n    loss = 0\n    correct = 0\n    \n    with torch.no_grad():\n        for data, target in dev_loader:\n            data = data.to(device)\n            target = target.to(device)\n\n            output = model(data)\n\n            loss += criterion(output, target, size_average=False).data\n\n            pred = output.data.max(1, keepdim=True)[1]\n            correct += pred.eq(target.max(1, keepdim=True)[1].data.view_as(pred)).cpu().sum().numpy()\n    \n    loss /= len(dev_loader.dataset)\n    accuracy = correct / len(dev_loader.dataset)\n    \n    if history is not None:\n        history.loc[epoch, 'dev_loss'] = loss.cpu().numpy()\n        history.loc[epoch, 'dev_accuracy'] = accuracy\n    \n    print('Dev loss: {:.4f}, Dev accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(dev_loader.dataset),\n        100. * accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"%%time\nimport gc\n\nhistory = pd.DataFrame()\n\nfor epoch in range(n_epochs):\n    torch.cuda.empty_cache()\n    gc.collect()\n    train(epoch, history)\n    evaluate(epoch, history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot losses"},{"metadata":{"trusted":true},"cell_type":"code","source":"history['train_loss'].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.dropna()['dev_loss'].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history.dropna()['dev_accuracy'].plot();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('max', history.dropna()['dev_accuracy'].max())\nprint('max in last 5', history.dropna()['dev_accuracy'].iloc[-5:].max())\nprint('avg in last 5', history.dropna()['dev_accuracy'].iloc[-5:].mean())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.eval()\npredictions = []\n\nfor data in tqdm(test_loader):\n    data = data.to(device)\n    output = model(data).max(dim=1)[1] # argmax\n    predictions += list(output.data.cpu().numpy())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":1}