{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Categorical Tabular Pytorch Classifier\n_By Nick Brooks, 2020-01-10_"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n# from torch.utils.data import Dataset, DataLoader\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\n\nfrom contextlib import contextmanager\nimport time\nimport gc\nnotebookstart = time.time()\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Time Each Process\n    \"\"\"\n    t0 = time.time()\n    yield\n    print('\\n[{}] done in {} Minutes'.format(name, round((time.time() - t0)/60,2)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 50\ndebug = False\n\nif debug:\n    nrow = 5000\nelse:\n    nrow = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"Load\"):\n    PATH = \"/kaggle/input/cat-in-the-dat-ii/\"\n    train = pd.read_csv(PATH + \"train.csv\", index_col = 'id', nrows = nrow)\n    test = pd.read_csv(PATH + \"test.csv\", index_col = 'id', nrows = nrow)\n    submission_df = pd.read_csv(PATH + \"sample_submission.csv\")\n    [print(x.shape) for x in [train, test, submission_df]]\n\n    traindex = train.index\n    testdex = test.index\n\n    y_var = train.target.copy()\n    print(\"Target Distribution:\\n\",y_var.value_counts(normalize = True).to_dict())\n\n    df = pd.concat([train.drop('target',axis = 1), test], axis = 0)\n    del train, test, submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with timer(\"FE 1\"):\n    drop_cols=[\"bin_0\"]\n\n    # Split 2 Letters; This is the only part which is not generic and would actually require data inspection\n    df[\"ord_5a\"]=df[\"ord_5\"].str[0]\n    df[\"ord_5b\"]=df[\"ord_5\"].str[1]\n    drop_cols.append(\"ord_5\")\n\n    xor_cols = []\n    nan_cols = []\n    for col in df.columns:\n        # NUll Values\n        tmp_null = df.loc[:,col].isnull().sum()\n        if tmp_null > 0:\n            print(\"{} has {} missing values.. Filling\".format(col, tmp_null))\n            nan_cols.append(col)\n            if df.loc[:,col].dtype == \"O\":\n                df.loc[:,col].fillna(\"NAN\", inplace=True)\n            else:\n                df.loc[:,col].fillna(-1, inplace=True)\n        \n        # Categories that do not overlap\n        train_vals = set(df.loc[traindex, col].unique())\n        test_vals = set(df.loc[testdex, col].unique())\n        \n        xor_cat_vals=train_vals ^ test_vals\n        if xor_cat_vals:\n            df.loc[df[col].isin(xor_cat_vals), col]=\"xor\"\n            print(\"{} has {} xor factors, {} rows\".format(col, len(xor_cat_vals),df.loc[df[col] == 'xor',col].shape[0]))\n            xor_cols.append(col)\n\n\n    # One Hot Encode None-Ordered Categories\n    ordinal_cols=['ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5a', 'day', 'month']\n    X_oh=df[df.columns.difference(ordinal_cols)]\n    oh1=pd.get_dummies(X_oh, columns=X_oh.columns, drop_first=True, sparse=True)\n    ohc1=oh1.sparse.to_coo()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.base import TransformerMixin\nfrom itertools import repeat\nimport scipy\n\nclass ThermometerEncoder(TransformerMixin):\n    \"\"\"\n    Assumes all values are known at fit\n    \"\"\"\n    def __init__(self, sort_key=None):\n        self.sort_key = sort_key\n        self.value_map_ = None\n    \n    def fit(self, X, y=None):\n        self.value_map_ = {val: i for i, val in enumerate(sorted(X.unique(), key=self.sort_key))}\n        return self\n    \n    def transform(self, X, y=None):\n        values = X.map(self.value_map_)\n        \n        possible_values = sorted(self.value_map_.values())\n        \n        idx1 = []\n        idx2 = []\n        \n        all_indices = np.arange(len(X))\n        \n        for idx, val in enumerate(possible_values[:-1]):\n            new_idxs = all_indices[values > val]\n            idx1.extend(new_idxs)\n            idx2.extend(repeat(idx, len(new_idxs)))\n            \n        result = scipy.sparse.coo_matrix(([1] * len(idx1), (idx1, idx2)), shape=(len(X), len(possible_values)), dtype=\"int8\")\n            \n        return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"other_classes = [\"NAN\", 'xor']\n\nwith timer(\"Thermometer Encoder\"):\n    thermos=[]\n    for col in ordinal_cols:\n        if col==\"ord_1\":\n            sort_key=(other_classes + ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster']).index\n        elif col==\"ord_2\":\n            sort_key= (other_classes + ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot']).index\n        elif col in [\"ord_3\", \"ord_4\", \"ord_5a\"]:\n            sort_key=str\n        elif col in [\"day\", \"month\"]:\n            sort_key=int\n        else:\n            raise ValueError(col)\n\n        enc=ThermometerEncoder(sort_key=sort_key)\n        thermos.append(enc.fit_transform(df[col]))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"ohc=scipy.sparse.hstack([ohc1] + thermos).tocsr()\ndisplay(ohc)\n\nX_sparse = ohc[:len(traindex)]\ntest_sparse = ohc[len(traindex):]\n\nprint(X_sparse.shape)\nprint(test_sparse.shape)\n\ndel ohc; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Test Split\nX_train, X_valid, y_train, y_valid = train_test_split(X_sparse, y_var, test_size=0.3)\n\n[print(table.shape) for table in [X_train, y_train, X_valid, y_valid]];","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class TabularDataset(torch.utils.data.Dataset):\n    def __init__(self, data, y=None):\n        self.n = data.shape[0]\n        self.y = y\n        self.X = data\n\n    def __len__(self):\n        return self.n\n\n    def __getitem__(self, idx):\n        if self.y is not None:\n            return [self.X[idx].toarray(), self.y.astype(float).values[idx]]\n        else:\n            return [self.X[idx].toarray()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = TabularDataset(data = X_train, y = y_train)\nvalid_dataset = TabularDataset(data = X_valid, y = y_valid)\nsubmission_dataset = TabularDataset(data = test_sparse, y = None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 4096\n\ntrain_loader = torch.utils.data.DataLoader(train_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=True)\n\nval_loader = torch.utils.data.DataLoader(valid_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=False)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_dataset, \n                                           batch_size=batch_size, \n                                           shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter(train_loader))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self, dropout = .15):\n        super().__init__()\n        self.dropout = dropout\n        \n        self.fc1 = nn.Linear(X_sparse.shape[1], 64)\n        self.d1 = nn.Dropout(p=self.dropout)\n        self.fc2 = nn.Linear(64, 128)\n        self.d2 = nn.Dropout(p=self.dropout)\n        self.fc3 = nn.Linear(128, 64)\n        self.d3 = nn.Dropout(p=self.dropout)\n        self.fc4 = nn.Linear(64, 1)\n        self.out_act = nn.Sigmoid()\n\n    def forward(self, x):\n        x = F.relu(self.fc1(x))\n        x = self.d1(x)\n        x = F.relu(self.fc2(x))\n        x = self.d2(x)\n        x = F.relu(self.fc3(x))\n        x = self.d3(x)\n        x = self.fc4(x)\n        x = self.out_act(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net = Net()\nnet.to(device)\n\nlearning_rate = 0.1\n# https://github.com/ncullen93/torchsample/blob/master/README.md\noptimizer = torch.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)\nscheduler = ReduceLROnPlateau(optimizer, min_lr = 0.00001, mode='min', factor=0.1, patience=2, verbose=True)\n\nEPOCHS = 50\n\ncriterion = nn.BCELoss()\n# optimizer = optim.Adam(net.parameters(), lr=0.0001)\n\nnn_output = []\npatience = 0\nmin_val_loss = np.Inf\n\nfor epoch in range(EPOCHS): # 3 full passes over the data\n    epoch_loss = 0\n    train_metric_score = 0\n    net.train()\n    \n    for data in train_loader:  # `data` is a batch of data\n        X, y = data[0].squeeze(1), data[1]  # X is the batch of features, y is the batch of targets.\n        net.zero_grad()  # sets gradients to 0 before loss calc. You will do this likely every step.\n        output = net(X.to(device).float()).squeeze()  # pass in the reshaped batch\n        tloss = criterion(output, y.to(device))  # calc and grab the loss value\n        tloss.backward()  # apply this loss backwards thru the network's parameters\n        optimizer.step()  # attempt to optimize weights to account for loss/gradients \n        \n        epoch_loss += tloss.item()\n        score = metrics.roc_auc_score(y, output.detach().cpu().numpy())\n        train_metric_score += score\n    \n    # Evaluation with the validation set\n    net.eval() # eval mode\n    val_loss = 0\n    val_metric_score = 0\n    with torch.no_grad():\n        for data in val_loader:\n            X, y = data[0].squeeze(1), data[1]\n            \n            preds = net(X.to(device).float()).squeeze() # get predictions\n            vloss = criterion(preds.float(), y.to(device)) # calculate the loss\n            \n            score = metrics.roc_auc_score(y, preds.detach().cpu().numpy())\n            val_metric_score += score\n            val_loss += vloss.item()\n    \n    final_val_loss = val_loss/len(val_loader)\n    tmp_nn_output = [epoch + 1,EPOCHS,\n                     epoch_loss/len(train_loader),\n                     train_metric_score/len(train_loader),\n                     final_val_loss,\n                     val_metric_score/len(val_loader)\n                     ]\n    nn_output.append(tmp_nn_output)\n    \n    # Early Stopping\n    scheduler.step(final_val_loss)\n    \n    # Print the loss and accuracy for the validation set\n    print('Epoch [{}/{}] train loss: {:.4f} train metric: {:.4f} valid loss: {:.4f} val metric: {:.4f}'\n        .format(*tmp_nn_output))\n    \n    if min_val_loss > round(final_val_loss,4) :\n        min_val_loss = round(final_val_loss,4)\n        patience = 0\n    else:\n        patience += 1\n    \n    if patience > 6:\n        print(\"Early Stopping..\")\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_results = pd.DataFrame(nn_output,\n    columns = ['epoch','total_epochs','train_loss','train_metric','valid_loss','valid_metric']\n                         )\ndisplay(pd_results)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(12, 4))\naxes[0].plot(pd_results['epoch'],pd_results['valid_loss'], label='validation_loss')\naxes[0].plot(pd_results['epoch'],pd_results['train_loss'], label='train_loss')\n# axes[0].plot(pd_results['epoch'],pd_results['test_loss'], label='test_loss')\naxes[0].set_title(\"Loss\")\n\naxes[0].legend()\n\naxes[1].plot(pd_results['epoch'],pd_results['valid_metric'], label='Val')\naxes[1].plot(pd_results['epoch'],pd_results['train_metric'], label='Train')\n# axes[1].plot(pd_results['epoch'],pd_results['test_acc'], label='test_acc')\naxes[1].set_title(\"Roc_AUC Score\")\naxes[1].legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"net.eval() # Safety first\npredictions = torch.Tensor().to(device) # Tensor for all predictions\n\n# Go through the test set, saving the predictions in... 'predictions'\nfor X in submission_loader:\n#     preds = net(X)\n    preds = net(X[0].to(device).float()).squeeze()\n    predictions = torch.cat((predictions, preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({'id': testdex, 'target': predictions.cpu().detach().numpy()})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}