{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":29844,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nimport torch\nimport random\nimport numpy as np\nimport pandas as pd\nimport torch.nn as nn\nimport torch.optim as optim\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nfrom xgboost import XGBClassifier\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PATH = \"/kaggle/input/digit-recognizer/\"\n\ntorch.manual_seed(2020)\nnp.random.seed(2020)\nrandom.seed(2020)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nif device.type == \"cuda\":\n    torch.cuda.get_device_name()","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"embedding_dims = 2\nbatch_size = 32\nepochs = 50","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(PATH+\"train.csv\")\ntest_df = pd.read_csv(PATH+\"test.csv\")\n\ntrain_df.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define class MNIST\nI'll define `class MNIST` which was inherited `torch.utils.data.Dataset`.","metadata":{}},{"cell_type":"code","source":"class MNIST(Dataset):\n    def __init__(self, df, train=True, transform=None):\n        self.is_train = train\n        self.transform = transform\n        self.to_pil = transforms.ToPILImage()\n        \n        if self.is_train:            \n            self.images = df.iloc[:, 1:].values.astype(np.uint8)\n            self.labels = df.iloc[:, 0].values\n            self.index = df.index.values\n        else:\n            self.images = df.values.astype(np.uint8)\n        \n    def __len__(self):\n        return len(self.images)\n    \n    def __getitem__(self, item):\n        anchor_img = self.images[item].reshape(28, 28, 1)\n        \n        if self.is_train:\n            anchor_label = self.labels[item]\n\n            positive_list = self.index[self.index!=item][self.labels[self.index!=item]==anchor_label]\n\n            positive_item = random.choice(positive_list)\n            positive_img = self.images[positive_item].reshape(28, 28, 1)\n            \n            negative_list = self.index[self.index!=item][self.labels[self.index!=item]!=anchor_label]\n            negative_item = random.choice(negative_list)\n            negative_img = self.images[negative_item].reshape(28, 28, 1)\n            \n            if self.transform:\n                anchor_img = self.transform(self.to_pil(anchor_img))\n                positive_img = self.transform(self.to_pil(positive_img))\n                negative_img = self.transform(self.to_pil(negative_img))\n            \n            return anchor_img, positive_img, negative_img, anchor_label\n        \n        else:\n            if self.transform:\n                anchor_img = self.transform(self.to_pil(anchor_img))\n            return anchor_img\n        ","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_ds = MNIST(train_df, \n                 train=True,\n                 transform=transforms.Compose([\n                     transforms.ToTensor()\n                 ]))\ntrain_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=4)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_ds = MNIST(test_df, train=False, transform=transforms.ToTensor())\ntest_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=4)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define TripletLoss","metadata":{}},{"cell_type":"code","source":"class TripletLoss(nn.Module):\n    def __init__(self, margin=1.0):\n        super(TripletLoss, self).__init__()\n        self.margin = margin\n        \n    def calc_euclidean(self, x1, x2):\n        return (x1 - x2).pow(2).sum(1)\n    \n    def forward(self, anchor: torch.Tensor, positive: torch.Tensor, negative: torch.Tensor) -> torch.Tensor:\n        distance_positive = self.calc_euclidean(anchor, positive)\n        distance_negative = self.calc_euclidean(anchor, negative)\n        losses = torch.relu(distance_positive - distance_negative + self.margin)\n\n        return losses.mean()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define Neural Network","metadata":{}},{"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self, emb_dim=128):\n        super(Network, self).__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(1, 32, 5),\n            nn.PReLU(),\n            nn.MaxPool2d(2, stride=2),\n            nn.Dropout(0.3),\n            nn.Conv2d(32, 64, 5),\n            nn.PReLU(),\n            nn.MaxPool2d(2, stride=2),\n            nn.Dropout(0.3)\n        )\n        \n        self.fc = nn.Sequential(\n            nn.Linear(64*4*4, 512),\n            nn.PReLU(),\n            nn.Linear(512, emb_dim)\n        )\n        \n    def forward(self, x):\n        x = self.conv(x)\n        x = x.view(-1, 64*4*4)\n        x = self.fc(x)\n        # x = nn.functional.normalize(x)\n        return x","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### weight initialize","metadata":{}},{"cell_type":"code","source":"def init_weights(m):\n    if isinstance(m, nn.Conv2d):\n        torch.nn.init.kaiming_normal_(m.weight)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create Instances\nUse JIT compilation for high speed.","metadata":{}},{"cell_type":"code","source":"model = Network(embedding_dims)\nmodel.apply(init_weights)\nmodel = torch.jit.script(model).to(device)\n\noptimizer = optim.Adam(model.parameters(), lr=0.001)\ncriterion = torch.jit.script(TripletLoss())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"model.train()\nfor epoch in tqdm(range(epochs), desc=\"Epochs\"):\n    running_loss = []\n    for step, (anchor_img, positive_img, negative_img, anchor_label) in enumerate(tqdm(train_loader, desc=\"Training\", leave=False)):\n        anchor_img = anchor_img.to(device)\n        positive_img = positive_img.to(device)\n        negative_img = negative_img.to(device)\n        \n        optimizer.zero_grad()\n        anchor_out = model(anchor_img)\n        positive_out = model(positive_img)\n        negative_out = model(negative_img)\n        \n        loss = criterion(anchor_out, positive_out, negative_out)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss.append(loss.cpu().detach().numpy())\n    print(\"Epoch: {}/{} - Loss: {:.4f}\".format(epoch+1, epochs, np.mean(running_loss)))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Save Params","metadata":{}},{"cell_type":"code","source":"torch.save({\"model_state_dict\": model.state_dict(),\n            \"optimzier_state_dict\": optimizer.state_dict()\n           }, \"trained_model.pth\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_results = []\nlabels = []\n\nmodel.eval()\nwith torch.no_grad():\n    for img, _, _, label in tqdm(train_loader):\n        train_results.append(model(img.to(device)).cpu().numpy())\n        labels.append(label)\n        \ntrain_results = np.concatenate(train_results)\nlabels = np.concatenate(labels)\ntrain_results.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualization ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15, 10), facecolor=\"azure\")\nfor label in np.unique(labels):\n    tmp = train_results[labels==label]\n    plt.scatter(tmp[:, 0], tmp[:, 1], label=label)\n\nplt.legend()\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training XGBoost Model","metadata":{}},{"cell_type":"code","source":"tree = XGBClassifier(seed=2020)\ntree.fit(train_results, labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_results = []\ntest_labels = []\n\nmodel.eval()\nwith torch.no_grad():\n    for img in tqdm(test_loader):\n        test_results.append(model(img.to(device)).cpu().numpy())\n        \ntest_results = np.concatenate(test_results)\n\nplt.figure(figsize=(15, 10), facecolor=\"azure\")\nplt.scatter(test_results[:, 0], test_results[:, 1], label=label)\n\ntest_results.shape","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"code","source":"submit = pd.read_csv(PATH+\"sample_submission.csv\")\nsubmit.Label = tree.predict(test_results)\n\nsubmit.head()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submit.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}