{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5"},"cell_type":"markdown","source":"# Official example\n## Neural network baseline\n\nNeural networks require a lot more compute time. It is much faster to train them on GPU. This is how to enable GPU at Kernel: https://www.kaggle.com/dansbecker/running-kaggle-kernels-with-a-gpu"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import sys\nimport os\nimport csv\nimport time\ncsv.field_size_limit(sys.maxsize)  # needed for torchtext\n\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torchtext\nfrom tqdm import tqdm\n\nimport sklearn.metrics as skm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"095f0d4a83b2f06bf87d6b3380975c5c72a36c57","trusted":false},"cell_type":"code","source":"assert torch.cuda.is_available(), 'We strongly reccomend using GPU for this kernel'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe512b0b142d7f7f95e2fb4c2976803ae079bd86","trusted":false},"cell_type":"code","source":"MAX_TITLE_LEN = 20\nMAX_BODY_LEN = 1000\n\nindex2label = ['news', 'clickbait', 'other']\nlabel2index = {l: i for i, l in enumerate(index2label)}\n\ntitle_field = torchtext.data.Field(lower=True, include_lengths=False, fix_length=MAX_TITLE_LEN, batch_first=True)\nbody_field = torchtext.data.Field(lower=True, include_lengths=False, fix_length=MAX_BODY_LEN, batch_first=True)\nlabel_field = torchtext.data.Field(sequential=False, is_target=True, use_vocab=False,\n                                   preprocessing=lambda x: label2index[x])\n\ntrain_dataset = torchtext.data.TabularDataset('../input/train.csv',\n                                              format='csv',\n                                              fields={'title': ('title', title_field),\n                                                      'text': ('body', body_field),\n                                                      'label': ('label', label_field)})\n\nval_dataset = torchtext.data.TabularDataset('../input/valid.csv',\n                                            format='csv',\n                                            fields={'title': ('title', title_field),\n                                                    'text': ('body', body_field),\n                                                    'label': ('label', label_field)})\n\ntest_dataset = torchtext.data.TabularDataset('../input/test.csv',\n                                            format='csv',\n                                            fields={'title': ('title', title_field),\n                                                    'text': ('body', body_field)})\n\nbody_field.build_vocab(train_dataset, min_freq=2)\nlabel_field.build_vocab(train_dataset)\nvocab = body_field.vocab\ntitle_field.vocab = vocab\n\nprint('Vocab size: ', len(vocab))\nprint(train_dataset[0].title)\nprint(train_dataset[0].body[:15])\nprint(train_dataset[0].label)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"158bfa85e1d5185b8ff33e7c47f1199d7d88b6c9","trusted":false},"cell_type":"code","source":"train_loader, val_loader = torchtext.data.Iterator.splits((train_dataset, val_dataset),\n                                                           batch_sizes=(64, 64),\n                                                           sort=False,\n                                                           device='cuda')\n\nbatch = next(iter(train_loader))\nprint(batch)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db7db1595a4020677266083ac1f408fa2478de4c"},"cell_type":"markdown","source":"## Network architecture"},{"metadata":{"_uuid":"d95008ce48ce29f6c20f06f05ddca67d0eda2b33","trusted":false},"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self, vocab_size, embedding_size, n_classes,\n                 kernel_sizes_cnn, filters_cnn: int, dense_size: int,\n                 dropout_rate: float = 0.,):\n        super().__init__()\n\n        self._n_classes = n_classes\n        self._vocab_size = vocab_size\n        self._embedding_size = embedding_size\n        self._kernel_sizes_cnn = kernel_sizes_cnn\n        self._filters_cnn = filters_cnn\n        self._dense_size = dense_size\n        self._dropout_rate = dropout_rate\n\n        self.embedding = nn.Embedding(vocab_size, embedding_size, padding_idx=0)\n\n        self.cnns = []\n        for i in range(len(kernel_sizes_cnn)):\n            in_channels = embedding_size\n\n            cnn = nn.Sequential(\n                nn.Conv1d(in_channels, filters_cnn, kernel_sizes_cnn[i]),\n                nn.BatchNorm1d(filters_cnn),\n                nn.ReLU()\n            )\n            cnn.apply(self.init_weights)\n\n            self.add_module(f'cnn_{i}', cnn)\n            self.cnns.append(cnn)\n        \n        # concatenated to hidden to classes\n        self.projection = nn.Sequential(\n            nn.Dropout(dropout_rate),\n            nn.Linear(filters_cnn * len(kernel_sizes_cnn), dense_size),\n            nn.BatchNorm1d(dense_size),\n            nn.ReLU(),\n            nn.Dropout(dropout_rate),\n            nn.Linear(dense_size, n_classes)\n        )\n\n    @staticmethod\n    def init_weights(module):\n        if type(module) == nn.Linear or type(module) == nn.Conv1d:\n            nn.init.kaiming_normal_(module.weight)\n\n    def forward(self, x):\n        x0 = self.embedding(x)\n        x0 = torch.transpose(x0, 1, 2)\n\n        outputs0 = []\n        outputs1 = []\n\n        for i in range(len(self.cnns)):\n            cnn = getattr(self, f'cnn_{i}')\n            # apply cnn and global max pooling\n            pooled, _ = cnn(x0).max(dim=2)\n            outputs0.append(pooled)\n\n        x0 = torch.cat(outputs0, dim=1) if len(outputs0) > 1 else outputs0[0]\n        return self.projection(x0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7d3c0e704ebd3d8e56cebbd316c444bf7581e5dd","trusted":false},"cell_type":"code","source":"model = CNN(vocab_size=len(vocab), embedding_size=300, n_classes=3, kernel_sizes_cnn=(1, 2, 3, 5),\n            filters_cnn=512, dense_size=256, dropout_rate=0.5)\nmodel.cuda()  # move model to GPU\nmodel","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02489b79bf67a48ccb38825ba0c9b5042cf84904"},"cell_type":"markdown","source":"## Training loop"},{"metadata":{"_uuid":"bd465c8eedaadc63944f0b3d8d554296ebb25a7d","trusted":false},"cell_type":"code","source":"epochs = 3\noptimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n\nloss_f = F.cross_entropy\n\nfor epoch in tqdm(range(epochs)):\n    for i, batch in enumerate(train_loader):\n        (title, body), label = batch\n\n        logits = model(title)\n        loss = loss_f(logits, label)\n\n        loss.backward()\n        optimizer.step()\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"db11776a3ea012e194148cbbe3ffd7b1881775f2"},"cell_type":"markdown","source":"## Test model on validation set"},{"metadata":{"_uuid":"ad74388da70b2bf1203d1dacbeecea6c6b4deff6","trusted":false},"cell_type":"code","source":"predictions = []\nlabels = []\n\n# change model mode to 'evaluation'\n# disable dropout and use learned batch norm statistics\nmodel.eval()\n\nwith torch.no_grad():\n    for batch in val_loader:\n        (title, body), label = batch\n        logits = model(title)\n\n        y_pred = torch.max(logits, dim=1)[1]\n        # move from GPU to CPU and convert to numpy array\n        y_pred_numpy = y_pred.cpu().numpy()\n\n        predictions = np.concatenate([predictions, y_pred_numpy])\n        labels = np.concatenate([labels, label.cpu().numpy()])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2249aca36910b475fb07dcbf67ab58f3c081a95e","trusted":false},"cell_type":"code","source":"skm.f1_score(labels, predictions, average='macro')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7cc7270d3f85d12014b24f17ee91bc34f1a5f2e"},"cell_type":"markdown","source":"It can be really hard to beat SVM at text classification, but it is almost always possible with neural network.\nThis neural network may have slightly worse metrics than SVM. However, you can tweak hyperparameters and number of epochs or to change network architecture to get better results."},{"metadata":{"_uuid":"6e4277f492131dfa66c506336e21f3d608f4e3b5"},"cell_type":"markdown","source":"## Make predictions on test set"},{"metadata":{"_uuid":"bacd4a023f01ea55f01c93319f8c6400e49be5e1","trusted":false},"cell_type":"code","source":"# Do not shuffle test set! You need id to label mapping\ntest_loader = torchtext.data.Iterator(test_dataset, batch_size=128, device='cuda', shuffle=False)\n\npredictions = []\n\nmodel.eval()\n\nwith torch.no_grad():\n    for batch in test_loader:\n        (title, body), label = batch\n        logits = model(title)\n\n        y_pred = torch.max(logits, dim=1)[1]\n        # move from GPU to CPU and convert to numpy array\n        y_pred_numpy = y_pred.cpu().numpy()\n\n        predictions = np.concatenate([predictions, y_pred_numpy])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cde33292cd94cce332f1028af2656d7affaaaabb","trusted":true},"cell_type":"code","source":"predictions_str = [index2label[int(p)] for p in predictions]\n\n# test.csv index in a contiguous integers from 0 to len(test_set)\n# to this should work fine\nsubmission = pd.DataFrame({'id': list(range(len(predictions_str))), 'label': predictions_str})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}