{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport os\nimport torch\nimport torch.nn as nn\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score as auc\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"TRAIN_PATH = '/kaggle/input/cat-in-the-dat-ii/train.csv'\nTEST_PATH = '/kaggle/input/cat-in-the-dat-ii/test.csv'\n\ntrain_df = pd.read_csv(TRAIN_PATH)\ntest_df = pd.read_csv(TEST_PATH)\n\n#train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add a target column with -1 in each instance.\ntest_df.loc[:, 'target'] = -1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = pd.concat([train_df, test_df]).reset_index(drop=True)\ndata.shape, train_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [f for f in train_df.columns if f not in ['id', 'target']]\nprint(features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I've the features now. Let's do the Label Encoding and then we would do entity encoding\n# for each feature in features we fit and return encoded labels \ndef label_encoder(data, features):\n    for feat in features:\n        le = LabelEncoder()\n        data.loc[:, feat] = le.fit_transform(data[feat].astype(str).fillna('-1').values)\n        \n    return data\n\ndata = label_encoder(data, features)\ntrain = data[:500000]\nvalid = data[500000:600000]\ntest = data[600000:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape, train.shape, valid.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Always state what you want from a function\n# This function will return embedding for all of the categorical features\ndef entity_embedding(df, col):\n    inputs = []\n    vector_size = 0\n    for c in col:\n        unique_values = df[c].nunique()\n        vector_size = vector_size + int(min(np.ceil(unique_values / 2), 64))\n        #embedding = nn.Embedding(len(df), int(min(np.ceil(unique_values / 2), 10)))\n        #embed = embedding(torch.tensor((df[c].values), dtype=torch.long))\n        #inputs.append(embed)\n    return vector_size #, inputs\n\nvector_size = entity_embedding(train, features)\n\n# concat each vector along it's dim=1\n#x = torch.cat([e for e in entEmb], dim=1)\n\n#x.size(), \nvector_size","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I need a module/function/class that takes in a LabelEncoded df and gives out an Embedding provided I can \ntrain the embedding as well."},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset(torch.utils.data.Dataset):\n    def __init__(self, df, features):\n        self.df = df.drop(['id', 'target'], axis=1).values\n        self.target = df.target.values\n        self.features = features\n        self.unique_values = [int(df[feat].nunique()) for feat in self.features]\n        \n    def __len__(self):\n        return len(self.df)    \n    \n    def __getitem__(self, idx):\n        inputs = self.df[idx]\n        targets = self.target[idx]\n        unique_vals = self.unique_values\n        sample = {'inputs': inputs, 'targets': targets, 'unique': unique_vals}\n        \n        return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = Dataset(train, features)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# In this cell we're trying to create a list of embeddings\nunique_vals = dataset[:]['unique']\nemb_list = [nn.Embedding(4, int(min(np.ceil(val / 2), 50))) for val in unique_vals]\nfor idx, emb in zip(list([i for i in range(23)]), emb_list):\n    print(f\"Index: {idx},    {emb}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# list of embeddings vector\n# [emb(labels_idx) for idx, emb zip(index, emb_List)]\n#emb = [emb(torch.tensor((dataset[0:4]['inputs'][:, idx]), dtype=torch.long)) for idx, emb in zip(list([i for i in range(23)]),emb_list)]\n#emb = torch.cat([e for e in emb], dim=1)\n#emb.size()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This class will take in the input from dataset and \n# returns the flattened vector\n\nclass EmbeddingLayer(nn.Module):\n    def __init__(self, unique_vals):\n        super(EmbeddingLayer, self).__init__()\n        \n        self.embed_list = [nn.Embedding(100000, int(min(np.ceil(val / 2), 64))) for val in unique_vals]\n            \n    def forward(self, inputs):\n        emb = [emb(torch.tensor((inputs[:, idx]), dtype=torch.long)) for idx, emb in zip(list([i for i in range(23)]), self.embed_list)]\n        out = torch.cat([e for e in emb], dim=1)\n        return out\n\nprint(dataset[0:4]['inputs'].shape)\nembb = EmbeddingLayer(unique_vals)\nembb(dataset[0:4]['inputs']).size()       ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainset = Dataset(train, features)\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True)\n\nvalidset = Dataset(valid, features)\nvalidloader = torch.utils.data.DataLoader(validset, batch_size=128, shuffle=False)\n\ntestset = Dataset(test, features)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=64)\n\nlen(trainloader), len(validloader), len(testloader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self, input_size, unique_vals, device, hidden_size=1024, dropout=0.5):\n        super(Model, self).__init__()\n        \n        self.fc1 = nn.Linear(input_size, hidden_size)\n        self.fc2 = nn.Linear(hidden_size, hidden_size)\n        self.fc3 = nn.Linear(hidden_size, 1)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(dropout)\n        self.embedding = EmbeddingLayer(unique_vals)\n        self.sigmoid = nn.Sigmoid()\n        self.device = device\n        \n        self.bn1 = nn.BatchNorm1d(hidden_size)\n        self.bn2 = nn.BatchNorm1d(hidden_size)\n        \n    def forward(self, le_df):\n        embed = self.embedding(le_df)\n        embed = embed.to(self.device)\n        out = self.bn1(self.relu(self.fc1(embed)))\n        \n        out = self.dropout(out)\n        \n        out = self.bn2(self.relu(self.fc2(out)))\n        out = self.dropout(out)\n        \n        out = self.fc3(out)\n        #print(out.size(), '------')\n        return self.sigmoid(out)\n \n\nmodel = Model(vector_size, unique_vals, device).to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for data in trainloader:\n    inp, trg = data['inputs'], data['targets']\n    print(inp.size(), trg.size())\n    break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# important series of operations to free tensor from cuda() and convert into simple numpy() array\ntorch.randn(3, 1).to(device).detach().cpu().numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(dataloader, model, criterion, optimizer, auc, device):\n    model.train()\n    \n    prev_auc = []\n    \n    print('Training: \\n')\n    \n    for i, data in enumerate(dataloader):\n        inputs, target = data['inputs'], data['targets']\n        \n        inputs = inputs.to(device)\n        target = torch.tensor(target, dtype=torch.float)\n        target = target.to(device)\n        #print(inputs.size(), target.size())\n        optimizer.zero_grad()\n        output = model(inputs)\n        #print(f\"Output: {output.squeeze(1).size()}\")\n        #print(f\"Target: {target.size()}\")\n        loss = criterion(output, target)\n        auc_score = auc(target.detach().cpu().numpy(), output.detach().cpu().numpy())\n        #print(target.detach().cpu().numpy().shape)\n        loss.backward()\n        optimizer.step()\n        \n        if i % 400 == 0:\n            print(f\"bi: {i},  loss: {loss.item():.4f},  auc: {auc_score:.4f}\")\n        \n            if len(prev_auc) == 0:\n                prev_auc.append(auc_score)\n\n            if (len(prev_auc) > 0) and (auc_score > max(prev_auc)):\n                prev_auc.append(auc_score)\n                torch.save(model, f'model{len(prev_auc)}.pth')\n        \n    return loss.item()\n\n\ndef evaluate(dataloader, model, criterion, optimizer, auc, device):\n    model.eval()\n    \n    scores = []\n    print('\\n')\n    print('Validation: \\n')\n    for i, data in enumerate(dataloader):\n        inputs, target = data['inputs'], data['targets']\n        \n        inputs = inputs.to(device)\n        target = torch.tensor(target, dtype=torch.float)\n        target = target.to(device)\n        optimizer.zero_grad()\n        output = model(inputs)\n        loss = criterion(output, target)\n        auc_score = auc(target.detach().cpu().numpy(), output.detach().cpu().numpy())\n        scores.append(auc_score)\n        \n        if i % 100 == 0:\n            print(f\"bi: {i},  loss: {loss.item():.4f},  auc: {auc_score:.4f}\")\n\n    return loss.item(), np.mean(scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.BCELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.0005)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for epoch in range(1):\n    train_loss = train(trainloader, model, criterion, optimizer, auc, device)\n    val_loss, val_acc = evaluate(validloader, model, criterion, optimizer, auc, device)\n    print(f\"Epoch: {epoch+1}/10, train_loss: {train_loss:.4f}, val_loss: {val_loss:.4f}, auc: {val_acc:.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load saved model\nmodel = torch.load('model2.pth')\n\n# perform validation\nval_loss, val_acc = evaluate(validloader, model, criterion, optimizer, auc, device)\n\nprint(f\"val_loss: {val_loss:.4f}, auc: {val_acc:.4f}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prediction for test set for submission\ndef test(dataloader, model, device):\n    model.eval()\n    \n    print('\\n')\n    print('Validation: \\n')\n    \n    predictions = []\n    \n    for i, data in enumerate(dataloader):\n        inputs, target = data['inputs'], data['targets']\n        \n        inputs = inputs.to(device)\n        target = torch.tensor(target, dtype=torch.float)\n        target = target.to(device)\n        optimizer.zero_grad()\n        \n        output = model(inputs)\n        \n        predictions.append(output.detach().cpu().numpy())\n    print('Prediciton complete!')\n    return predictions\n\npredictions = test(testloader, model, device)\nlen(predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# flatten out the list of lists\npred_list = []\nfor pred in predictions:\n    for p in pred:\n        pred_list.append(p)\n\nlen(pred_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB_PATH = '/kaggle/input/cat-in-the-dat-ii/sample_submission.csv'\nsubmission_df = pd.read_csv(SUB_PATH)\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SUB_PATH = '/kaggle/input/cat-in-the-dat-ii/sample_submission.csv'\nsubmission_df = pd.read_csv(SUB_PATH)\nsubmission_df['target'] = pd.DataFrame(pred_list)\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.read_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}