{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Inspired by [Abhishek's Kernel](https://www.kaggle.com/abhishek/same-old-entity-embeddings), I just tried to retranslate handling categorical features in simple way using Pytorch"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport os\nimport copy\nimport traceback\nimport datetime\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    '''\n    Reduce file memory usage\n    Source: https://www.kaggle.com/artgor\n    \n    Parameters:\n    -----------\n    df: DataFrame\n        Dataset on which to perform transformation\n    verbose: bool\n        Print additional information\n    Returns:\n    --------\n    DataFrame\n        Dataset as pandas DataFrame\n    '''\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                c_prec = df[col].apply(lambda x: np.finfo(x).precision).max()\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16)\\\n                                               .max and c_prec == np.finfo(np.float16).precision:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32)\\\n                                                .max and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'\\\n                                               .format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    \n    return (df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CatDataset(Dataset):\n    def __init__(self, data, cat_cols=None, output_col=None, train=True):\n        self.n = data.shape[0]\n\n        if output_col:\n            self.y = data[output_col].astype(np.float32).values\n        else:\n            self.y =  np.zeros((self.n, 1))\n\n        self.cat_cols = cat_cols if cat_cols else []\n\n        if self.cat_cols:\n            self.cat_X = data[cat_cols].astype(np.int64).values\n        else:\n            self.cat_X =  np.zeros((self.n, 1))\n\n    def __len__(self):\n        \"\"\"\n        Denotes the total number of samples.\n        \"\"\"\n        return self.n\n\n    def __getitem__(self, idx):\n        \"\"\"\n        Generates one sample of data.\n        \"\"\"\n        return [self.y[idx], self.cat_X[idx]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class GottaTorch(nn.Module):\n\n    def __init__(self, emb_dims, lin_layer_sizes,\n               output_size, emb_dropout, lin_layer_dropouts):\n                \n        \"\"\"\n        emb_dims: List of two element tuples\n        For each categorical feature the first element of a tuple will\n        denote the number of unique values of the categorical\n        feature. The second element will denote the embedding\n        dimension to be used for that feature.\n        \"\"\"\n\n        super().__init__()\n\n        # Embedding layers\n        self.emb_layers = nn.ModuleList([nn.Embedding(x, y) for x, y in emb_dims])   \n        self.no_of_embs = sum([y for x, y in emb_dims])\n\n        # Linear Layers\n        first_lin_layer = nn.Linear(in_features=self.no_of_embs, \n                                    out_features=lin_layer_sizes[0])\n\n        self.lin_layers =\\\n         nn.ModuleList([first_lin_layer] +\\\n              [nn.Linear(lin_layer_sizes[i], lin_layer_sizes[i + 1])\n               for i in range(len(lin_layer_sizes) - 1)])\n\n        for lin_layer in self.lin_layers:\n            nn.init.kaiming_normal_(lin_layer.weight.data)\n\n        # Output Layer\n        self.output_layer = nn.Linear(lin_layer_sizes[-1],\n                                      output_size)\n        nn.init.kaiming_normal_(self.output_layer.weight.data)\n\n        # Batch Norm Layers\n        self.first_bn_layer = nn.BatchNorm1d(self.no_of_embs)\n        self.bn_layers = nn.ModuleList([nn.BatchNorm1d(size)\n                                        for size in lin_layer_sizes])\n\n        # Dropout Layers\n        self.emb_dropout_layer = nn.Dropout(emb_dropout) \n        self.droput_layers = nn.ModuleList([nn.Dropout(size)\n                                      for size in lin_layer_dropouts])\n        \n        self.softmax = nn.Softmax(dim=1)\n\n    def forward(self, cat_data):\n\n        if self.no_of_embs != 0:\n            x = [emb_layer(cat_data[:, i])\n               for i, emb_layer in enumerate(self.emb_layers)]\n            x = torch.cat(x, 1)\n            x = self.first_bn_layer(x)\n            x = self.emb_dropout_layer(x)       \n\n        for lin_layer, dropout_layer, bn_layer in\\\n            zip(self.lin_layers, self.droput_layers, self.bn_layers):\n\n            x = F.relu(lin_layer(x))\n            x = dropout_layer(x)\n            x = bn_layer(x)\n            \n        x = self.output_layer(x)\n        x = self.softmax(x)\n        \n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_with_model(model, dataset, device=None, batch_size=32, num_workers=0):\n    \n    if device is None:\n        device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    results_by_batch = []\n\n    device = torch.device(device)\n    model.to(device)\n    model.eval()\n\n    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)\n    with torch.no_grad():\n        import tqdm\n        for _, batch_x in tqdm.tqdm(dataloader, total=len(dataset)/batch_size):\n            batch_x = copy_data_to_device(batch_x, device)\n\n            batch_pred = model(batch_x)\n            results_by_batch.append(batch_pred.detach().cpu().numpy())\n            \n    return np.concatenate(results_by_batch, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_eval_loop(model, train_dataset, val_dataset, criterion,\n                    lr=1e-4, epoch_n=5, batch_size=1024,\n                    device=None, early_stopping_patience=10, l2_reg_alpha=0,\n                    max_batches_per_epoch_train=10000,\n                    max_batches_per_epoch_val=1000,\n                    data_loader_ctor=DataLoader,\n                    optimizer_ctor=None,\n                    lr_scheduler_ctor=None,\n                    shuffle_train=True,\n                    dataloader_workers_n=1):\n    \n    \"\"\"\n    Useful sourse: https://github.com/Samsung-IT-Academy/stepik-dl-nlp/blob/master/dlnlputils\n    \"\"\"\n    \n    if device is None:\n        device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    device = torch.device(device)\n    model.to(device)\n\n    if optimizer_ctor is None:\n        optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=l2_reg_alpha)\n    else:\n        optimizer = optimizer_ctor(model.parameters(), lr=lr)\n\n    if lr_scheduler_ctor is not None:\n        lr_scheduler = lr_scheduler_ctor(optimizer)\n    else:\n        lr_scheduler = None\n\n    train_dataloader = data_loader_ctor(train_dataset, batch_size=batch_size, shuffle=shuffle_train,\n                                        num_workers=dataloader_workers_n)\n    val_dataloader = data_loader_ctor(val_dataset, batch_size=batch_size, shuffle=False,\n                                      num_workers=dataloader_workers_n)\n\n    best_val_loss = float('inf')\n    auc_valid = 0\n    best_epoch_i = 0\n    best_model = copy.deepcopy(model)\n\n    for epoch_i in range(epoch_n):\n        try:\n            epoch_start = datetime.datetime.now()\n            print('epoch {}'.format(epoch_i + 1))\n\n            model.train()\n            mean_train_loss = 0\n            train_batches_n = 0\n            for batch_i, (batch_y, batch_x) in enumerate(train_dataloader):\n                if batch_i > max_batches_per_epoch_train:\n                    break\n\n                batch_x = copy_data_to_device(batch_x, device)\n                batch_y = copy_data_to_device(batch_y, device)\n\n                pred = model(batch_x)[:, 1]\n                loss = criterion(pred, batch_y)\n\n                model.zero_grad()\n                loss.backward()\n\n                optimizer.step()\n\n                mean_train_loss += float(loss)\n                train_batches_n += 1\n\n            mean_train_loss /= train_batches_n\n            print('epoch: {} iterations, {:0.2f} sec'.format(train_batches_n,\n                                                           (datetime.datetime.now() - epoch_start).total_seconds()))\n            print('mean loss on train: ', mean_train_loss)\n\n            model.eval()\n            mean_val_loss = 0\n            val_batches_n = 0\n\n            with torch.no_grad():\n                for batch_i, (batch_y, batch_x) in enumerate(val_dataloader):\n                    if batch_i > max_batches_per_epoch_val:\n                        break\n\n                    batch_x = copy_data_to_device(batch_x, device)\n                    batch_y = copy_data_to_device(batch_y, device)\n\n                    pred = model(batch_x)[:, 1] \n                    loss = criterion(pred, batch_y)\n\n                    mean_val_loss += float(loss)\n                    auc = roc_auc_score(batch_y.data.cpu().numpy(), pred.data.cpu().numpy())\n                    val_batches_n += 1\n\n            mean_val_loss /= val_batches_n\n            print('mean loss on validation', mean_val_loss)\n            print('valid batch auc --> %.5f'% auc)\n\n            if mean_val_loss < best_val_loss:\n                best_epoch_i = epoch_i\n                best_val_loss = mean_val_loss\n                best_model = copy.deepcopy(model)\n                print('New best model!')\n            elif epoch_i - best_epoch_i > early_stopping_patience:\n                print('The model has not improved over the past {} epochs, stop training'.format(\n                    early_stopping_patience))\n                break\n\n            if lr_scheduler is not None:\n                lr_scheduler.step(mean_val_loss)\n\n            print()\n        except KeyboardInterrupt:\n            print('Stopped by user')\n            break\n        except Exception as ex:\n            print('Training error: {}\\n{}'.format(ex, traceback.format_exc()))\n            break\n\n    return {\"loss\": best_val_loss,\n            \"model\": best_model}\n\n\ndef seed_everything(seed=100):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything()\n    \n\ndef copy_data_to_device(data, device):\n    if torch.is_tensor(data):\n        return data.to(device)\n    elif isinstance(data, (list, tuple)):\n        return [copy_data_to_device(elem, device) for elem in data]\n    raise ValueError('Invalid data type {}'.format(type(data)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/test.csv\")\nsubm = pd.read_csv(\"/kaggle/input/cat-in-the-dat-ii/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.loc[:, \"target\"] = -1\ndata = pd.concat([train, test]).reset_index(drop=True)\nprint(\"dim data: \", data.shape)\n\nfeatures = data.columns.difference([\"id\", \"target\"]).tolist()\ntarget = \"target\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_encoders = {}\nfor cat_col in features:\n    label_encoders[cat_col] = LabelEncoder()\n    data[cat_col] = label_encoders[cat_col].fit_transform(data[cat_col]\\\n                                                          .astype('category').cat.codes\\\n                                                          .fillna(-1).values)\n    \ndata = reduce_mem_usage(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_dims = [int(data[col].nunique()) for col in features]\nemb_dims = [(x, min(50, (x + 1) // 2)) for x in cat_dims]\n\nprint(\"cat_dims: \", cat_dims, ',\\n')\nprint(\"emb_dims: \", emb_dims, ',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = data[data.target != -1].reset_index(drop=True)\ntest = data[data.target == -1].reset_index(drop=True)\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df, valid_df = train_test_split(train, test_size=0.3, stratify=train.target)\nprint(train_df.shape, valid_df.shape)\n\ntrain_dataset = CatDataset(data=train_df, cat_cols=features, output_col=target)\nvalid_dataset = CatDataset(data=valid_df, cat_cols=features, output_col=target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = GottaTorch(emb_dims, lin_layer_sizes=[300, 300],\n                   output_size=2, emb_dropout=0.3,\n                   lin_layer_dropouts=[0.3, 0.3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 1024*2\nLR = 9e-3\nLOSS_FN = nn.BCELoss()\nLR_SCHEDULER = lambda optim: torch.optim.lr_scheduler.ReduceLROnPlateau(optim, patience=3, \n                                                                     factor=0.5, verbose=True)\n\nd = train_eval_loop(model, train_dataset, valid_dataset, lr=LR, criterion=LOSS_FN, \n                    lr_scheduler_ctor=LR_SCHEDULER, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = CatDataset(data=test, cat_cols=features, output_col=target)\nres = predict_with_model(d[\"model\"], test_dataset, batch_size=64*4, num_workers=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm.target = res[:, 1]\nsubm.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feel free to continue experimenting..."}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}