{"cells":[{"metadata":{"trusted":true,"_uuid":"490b981104f0e001f06772347f1a77a19ecb416e"},"cell_type":"code","source":"!pip install torchsummary","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10dc6168c78bffe190597d23394ff54e698792d4","trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision.models as models\nfrom torch.autograd import Variable\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport cv2\nfrom tqdm import tqdm_notebook as tqdm\nfrom sklearn.model_selection import train_test_split\nimport torchvision\nfrom torchvision import transforms\nimport math\nfrom PIL import Image\nfrom torchsummary import summary \nfrom tqdm import trange \n\n%matplotlib inline\n\nh, w = 96, 96 #better to start off with lower values, like 32 or 64\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\ntorch.backends.cudnn.benchmark = True #speeds up training in case of constant input size\n\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8db07f3dfad330731c1e340995dad80abe7d4293","trusted":false},"cell_type":"code","source":"#we resize images while saving their aspect ratio and pad them with 255 (white background) to h*w size\ndef resize (img):\n    im = Image.fromarray(img)\n    im.thumbnail((h, w), Image.ANTIALIAS)\n    im = np.array(im).astype('float32')\n    return np.expand_dims(np.pad(im, (((h - im.shape[0]) // 2, (h - im.shape[0] + 1) // 2), ((w - im.shape[1]) // 2, (w - im.shape[1] + 1) // 2)), 'constant', constant_values=255), axis = -1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d70fc17f961c7b79542dca0400e8d9b0d4aef225","trusted":false},"cell_type":"code","source":"data1 = np.load('../input/train-1.npy')\ndata2 = np.load('../input/train-2.npy')\ndata3 = np.load('../input/train-3.npy')\ndata4 = np.load('../input/train-4.npy')\ntrain = np.concatenate((data1, data2, data3, data4), axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c528ffdc5ae1d5d1340ba5cd70764812d1c062e1","trusted":false},"cell_type":"code","source":"#data visualisation\nplt.figure(figsize=(16, 20))\nfor n, (image, tag) in enumerate(train, 1):\n    if n > 64:\n        break\n    plt.subplot(8, 8, n)\n    plt.title(tag)\n    plt.imshow(image, cmap='gray')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2dff08b62560cb19009d44423295048046e8143","trusted":false},"cell_type":"code","source":"#switching from Unicode to 0-999\ndct = dict(zip(range(1000), np.unique(train[:, 1])))\ndct = {v:k for k,v in dct.items()}\n\ntrain[:, 1] = [dct[code] for code in train[:, 1]] ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bd7c9c37920e0dd2bb5dbbe562bbb167cf43031","trusted":false},"cell_type":"code","source":"#validation split (random_state for better reproducibility of results)\ntrain, test = train_test_split(train, test_size=0.2, random_state=113)\n\ntrain_x_np, train_y_np = train[:, 0], train[:, 1].astype('int32')\ntest_x_np, test_y_np = test[:, 0], test[:, 1].astype('int32')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f9a4f32edb38a2ee2833ad53b841638e05c6955","trusted":false},"cell_type":"code","source":"#garbage collector (frees some RAM)\nimport gc\ngc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"feabf686c3d2d23205c502d83c07133f34577f07","trusted":false},"cell_type":"code","source":"#custom dataset is a good practice (helps with data preprocessing, augmentation, etc.)\nclass Hieroglyph_data(Dataset):\n    def __init__(self, tX, tY = None,\n                 transform = transforms.Compose(\n                     [transforms.ToPILImage(), \n                      transforms.ToTensor(), \n                      transforms.Normalize(mean=(0.5,), std=(0.5,))]), train=True): \n                                           #normalizing in range [-1; 1] speeds up training, better than [0; 1]\n        \n        if train == True:\n            self.X = tX\n            self.y = torch.Tensor(tY).type(torch.LongTensor)\n            self.transform = transform\n            self.train = True\n            \n        else:\n            self.X = tX\n            self.y = None\n            self.transform = transform\n            self.train = False\n            \n    def __len__(self):\n        return self.X.shape[0]\n\n    def __getitem__(self, idx):\n        #preprocessing on demand trades computations for memory\n        if self.train == True:\n            return self.transform(resize(self.X[idx])), self.y[idx] \n        else:\n            return self.transform(resize(self.X[idx]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5aaf43cd790df4031e570e6c7ee64dd78e67370a","trusted":false},"cell_type":"code","source":"#data augmentation significantly boosts accuracy and prevents overfitting (RandomAffine does the augmentation)\ntrain_dataset = Hieroglyph_data(train_x_np, train_y_np, transform = transforms.Compose(\n                                [transforms.ToPILImage(), \n                                 transforms.RandomAffine(\n                                 degrees=(-10, 10), \n                                 translate=(.1, .1), \n                                 scale=(0.9, 1.1),\n                                 shear=(-10, 10)),\n                                 transforms.ToTensor(), \n                                 transforms.Normalize(mean=(0.5,), std=(0.5,))]))\n\ntest_dataset = Hieroglyph_data(test_x_np, test_y_np)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"664d254b99b3232b7fb54ca1d3a6da39781c2d8c","trusted":false},"cell_type":"code","source":"num_classes = 1000\nbatch_size = 512\nnum_epochs = 20\n\n#pin_memory speeds up data transfer from CPU to GPU\ntrain_loader = DataLoader(train_dataset, batch_size = batch_size, shuffle = True, pin_memory = True) \ntest_loader = DataLoader(test_dataset, batch_size = batch_size, shuffle = False, pin_memory = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af8b7392a2eebc0722b360ffe4888d8495dba1d5","trusted":false},"cell_type":"code","source":"class PabloNet(nn.Module):    \n    def __init__(self):\n        super(PabloNet, self).__init__()\n          \n        self.layers_conv = nn.Sequential(\n            nn.Conv2d(1, 32, kernel_size=7, stride=2),\n            nn.BatchNorm2d(32),\n            nn.PReLU(),\n            #nn.Dropout(0.2),\n            nn.Conv2d(32, 64, kernel_size=5, stride=1),\n            nn.BatchNorm2d(64),\n            nn.PReLU(),\n            #nn.Dropout(0.25),\n            nn.MaxPool2d(kernel_size=2),\n            \n            nn.Conv2d(64, 128, kernel_size=5, stride=1),\n            nn.BatchNorm2d(128),\n            nn.PReLU(),\n            #nn.Dropout(0.25),\n            nn.Conv2d(128, 256, kernel_size=3, stride=1),\n            nn.BatchNorm2d(256),\n            nn.PReLU(),\n            nn.MaxPool2d(kernel_size=2),\n            \n            #nn.Dropout(0.25),\n            nn.Conv2d(256, 512, kernel_size=3, stride=1),\n            nn.BatchNorm2d(512),\n            nn.PReLU(),\n            \n            nn.Conv2d(512, 1024, kernel_size=3, stride=1),\n            nn.BatchNorm2d(1024),\n            nn.PReLU(),\n            #nn.AvgPool2d(kernel_size=2),\n            #nn.Dropout(0.25),\n        )\n        \n        self.layers_dense = nn.Sequential(\n            nn.Linear(9216, 512),\n            nn.BatchNorm1d(512),\n            nn.PReLU(),\n            nn.Linear(512, 1000)\n        )\n        \n    def forward(self, x):\n        x = self.layers_conv(x)\n        x = x.view(x.size(0), -1)\n        x = self.layers_dense(x)\n        return x     \n    \nmodel = PabloNet()\n\nmodel = model.to(device)\n\n\n#summary from torchsummary library is a helpful feature that provides useful Keras-style model summary that helps\n#you to evaluate model size, check if everything is ok, and roughly estimate batchsize.\n#Install it with pip install torchsummary\n\nsummary(model, (1, h, w))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6ba1b8e027b1f51340dff46820798cb0794560ec","trusted":false},"cell_type":"code","source":"error = nn.CrossEntropyLoss()\n\nlearning_rate = 0.002\n\noptimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.5)\n#lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.85)\n#this scheduler leads to more smooth convergence yet it seems to me it works better on bigger models that require \n#more epochs to train, and performs worse otherwise.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b60d841fe763229430f5f1ca0306ada1b9390bb","trusted":false},"cell_type":"code","source":"def train(epoch):\n    model.train() #don't forget to switch between train and eval!\n    \n    running_loss = 0.0 #more accurate representation of current loss than loss.item()\n\n    for i, (images, labels) in enumerate(train_loader):\n        images, labels = images.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(images)\n\n        loss = error(outputs, labels)\n\n        loss.backward()\n\n        optimizer.step()\n\n        running_loss += loss.item()\n\n        if (i + 1)% 50 == 0:\n            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n                    epoch, (i+ 1) * len(images), len(train_loader.dataset),\n                    100. * (i + 1) / len(train_loader), running_loss / 50))\n            \n            running_loss = 0.0\n            \ndef evaluate(data_loader):\n    model.eval() \n    loss = 0\n    correct = 0\n    \n    with torch.no_grad():\n        for i, (images, labels) in enumerate(data_loader):\n            images, labels = images.to(device), labels.to(device)\n\n            outputs = model(images)\n\n            loss += F.cross_entropy(outputs, labels, reduction = 'sum').item()\n\n            pred = outputs.data.max(1, keepdim=True)[1]\n            correct += pred.eq(labels.data.view_as(pred)).cpu().sum()\n        \n    loss /= len(data_loader.dataset)\n        \n    print('\\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.3f}%)\\n'.format(\n        loss, correct, len(data_loader.dataset),\n        100. * correct / len(data_loader.dataset)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b1072e43e1d68c55266189ffd0328470b15d88f5","trusted":false},"cell_type":"code","source":"for epoch in range(num_epochs): \n    lr_scheduler.step()\n    train(epoch)\n    evaluate(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"14476d7c3d1a803d26b9631c340152777557698b","trusted":false},"cell_type":"code","source":"#loading train data\npred = np.load('../input/test.npy')\npred_dataset = Hieroglyph_data(pred, train = False)\npred_loader = torch.utils.data.DataLoader(pred_dataset, batch_size = batch_size, shuffle = False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aca139420dc0c04f4062076f1d56facf7721a98d","trusted":false},"cell_type":"code","source":"def prediciton(data_loader):\n    model.eval()\n    test_pred = torch.LongTensor()\n    \n    with torch.no_grad():\n        for i, images in enumerate(data_loader):\n            images = images.to(device)\n\n            outputs = model(images)\n\n            pred = outputs.cpu().data.max(1, keepdim=True)[1]\n\n            test_pred = torch.cat((test_pred, pred), dim=0)\n        \n    return test_pred","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18d93e79f4c49227ef3179a642fdf10ee927acc4","trusted":false},"cell_type":"code","source":"#preparing the predictions\npred_y = prediciton(pred_loader)\npred_y = pred_y.cpu().numpy().flatten()\n\ndct = {v:k for k,v in dct.items()} #reversing keys and values leads to easy inversion from 0-999 back to Unicode \n\npred_y = [dct[code] for code in pred_y]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46da802ed7e355968bfbc3fa760add2c238ea8f4","trusted":false},"cell_type":"code","source":"#creating a submission csv using pandas\nimport pandas as pd\nresults = pd.Series(pred_y, name=\"Category\", dtype = 'object')\n\nsubmission = pd.concat([pd.Series(range(1,83248), name = \"Id\", dtype='object'), results],axis = 1)\n\nsubmission.to_csv(\"pablo_submit.csv\",index=False) \n#providing key information in submisson name is actually very important as you may need to reuse these submissions\n#for blending, for example (it's a practice used in Kaggle competitions; you basically fit several models with \n#different architectures and then generate an \"average\" submission (it's mostly used in classification tasks, so\n#it's basically voting, usually weighted voting: the right label for the sample is chosen as a label with most \n#'votes' - weighted entries of every submission file. Weights are often chosen in accordance with performance on\n#public leaderboards. The more advanced technique of weight choice is automated search: we can use sklearn function\n#that finds optimum of a given function with n variables. We can use validation holdout, fit various models and \n#generate submissions for this validation holdout. As we know correct labels for the holdout, we can calculate\n#accuracy (or any other metrics used for submission evaluation) for blended submissions with given weights for \n#each of csvs. Then we optimize this function using sklearn and use found weights for the actual submission. Given\n#that the validation holdout is representative enough, this will give us a significant boost. The main problem \n#here is the need for validation holdout to be the same in every model, so it's used not as frequent as it should\n#be. Rough estimations based on LB score, or even equal weights will still give you a small boost in LB, and, more\n#importantly, it increase the robustness of your submission; you are less likely to lose places because of the \n#overfit on public LB.","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}