{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.listdir('../input/image-3jhn-chunk2/image_3jhn_chunk2/Image_3JHN_chunk2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/champs-scalar-coupling/train.csv')\ntest = pd.read_csv('../input/champs-scalar-coupling/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_path = [\n    '../input/image-3jhn-chunk1/image_3jhn_chunk1/Image_3JHN_chunk1/',\n    '../input/image-3jhn-chunk2/image_3jhn_chunk2/Image_3JHN_chunk2/'\n]\n\ndescription = []\nfor i, path in enumerate(image_path):\n    files = [f for f in os.listdir(path) if f.endswith('.pkl')]\n    ids = [int(f.split('_')[2].strip('.pkl')) for f in files]\n    \n    desc = train[train['id'].isin(ids)].copy()\n    desc['filename'] = path + desc['molecule_name'] + '_' + desc['id'].astype(str) + '.pkl'\n\n    description.append(desc)\n\ndescription = pd.concat(description)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"description.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\n\n# check if CUDA is available\ntrain_on_gpu = torch.cuda.is_available()\n\nif not train_on_gpu:\n    print('CUDA is not available.  Training on CPU ...')\nelse:\n    print('CUDA is available!  Training on GPU ...')\n\n# set seed\ntorch.manual_seed(0)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1. Dataloader\n source: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nimport os\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader\n\nclass CustomImageDataset(Dataset):\n   \n    def __init__(self, csv_file, transform=None):\n        self.df = csv_file # pd.read_csv(root_dir + csv_file)\n#         self.root_dir = root_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \n        img_name = self.df.iloc[idx]['filename']\n        \n#         img_name = os.path.join(\n#             self.root_dir,\n#             str(self.df.iloc[idx]['molecule_name']) + '_' + str(self.df.iloc[idx]['id']) + '.pkl'\n#         )\n        \n        with open (img_name, 'rb') as fp:\n            image = pickle.load(fp)\n        \n        for c in range(5):\n            image[c] = np.clip(image[c], 0, 255) / 255\n        \n        img = torch.from_numpy(np.array(image))\n        img = img.type(torch.FloatTensor)\n        \n        sample = {'image': img,\n                  'target': self.df.iloc[idx]['scalar_coupling_constant']}\n\n        if self.transform:\n            sample['image'] = self.transform(sample['image'])\n\n        return sample['image'], sample['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_dataset = CustomImageDataset(csv_file=description)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2. Cross validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\ngroup_kfold = GroupKFold(n_splits=5)\n\ndf = description.copy()\ndf.reset_index(drop=True, inplace=True)\n\nX = df[['id', 'molecule_name']].copy()\ny = df['scalar_coupling_constant']\ngroups = df['molecule_name'].unique()\n\nfolds = []\nfor train_idx, valid_idx in group_kfold.split(X, y, X['molecule_name']):\n    folds.append([train_idx, valid_idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index_fold = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from torchvision import datasets\nimport torchvision.transforms as transforms\nfrom torch.utils.data.sampler import SubsetRandomSampler\n\n# number of subprocesses to use for data loading\nnum_workers = 0\n# how many samples per batch to load\nbatch_size = 40\n\n# convert data to a normalized torch.FloatTensor\ntransform = transforms.Compose([\n    transforms.Normalize((0.5, 0.5, 0.5, 0.5, 0.5), (0.5, 0.5, 0.5, 0.5, 0.5))\n    ])\n\ntrain_data = CustomImageDataset(\n    csv_file=description\n#     transform=transform\n)\n\ntrain_idx, valid_idx = folds[index_fold][0], folds[index_fold][1]\n\n# define samplers for obtaining training and validation batches\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\n# prepare data loaders (combine dataset and sampler)\ntrain_loader = torch.utils.data.DataLoader(train_data,\n                                           batch_size=batch_size,\n                                           sampler=train_sampler,\n                                           num_workers=num_workers\n                                          )\nvalid_loader = torch.utils.data.DataLoader(train_data,\n                                           batch_size=batch_size, \n                                           sampler=valid_sampler,\n                                           num_workers=num_workers\n                                          )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNN(nn.Module):\n    \"\"\"CNN.\"\"\"\n\n    def __init__(self):\n        \"\"\"CNN Builder.\"\"\"\n        super(CNN, self).__init__()\n\n        self.conv_layer = nn.Sequential(\n\n            # Conv Layer block 1\n            nn.Conv2d(in_channels=5, out_channels=20, kernel_size=3, padding=1),\n            nn.BatchNorm2d(20),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_channels=20, out_channels=20, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout2d(p=0.1),\n\n            # Conv Layer block 2\n            nn.Conv2d(in_channels=20, out_channels=35, kernel_size=3, padding=1),\n            nn.BatchNorm2d(35),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_channels=35, out_channels=35, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout2d(p=0.1),\n\n            # Conv Layer block 3\n            nn.Conv2d(in_channels=35, out_channels=50, kernel_size=3, padding=1),\n            nn.BatchNorm2d(50),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_channels=50, out_channels=50, kernel_size=3, padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n            nn.Dropout2d(p=0.1),\n        )\n\n        self.fc_layer = nn.Sequential(\n#             nn.Dropout(p=0.3),\n            nn.Linear(800, 600),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p=0.3),\n            nn.Linear(600, 300),\n            nn.ReLU(inplace=True),\n            nn.Dropout(p=0.3),\n            nn.Linear(300, 1)\n        )\n\n    def forward(self, x):\n        x = self.conv_layer(x)\n        x = x.view(x.size(0), -1)\n        x = self.fc_layer(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import os\n# os.listdir('../input/model-1-fold3')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a complete CNN\nmodel = CNN()\nprint(model)\n\n# model.load_state_dict(torch.load('../input/model-1-fold4/model_best.pt'))\n\n# move tensors to GPU if CUDA is available\nif train_on_gpu:\n    model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch.optim as optim\n\n# specify loss function (categorical cross-entropy)\ncriterion = nn.MSELoss()\nmae = nn.L1Loss()\n\n# specify optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of epochs to train the model\nimport time\nn_epochs = 150\nloss_per_iter = []\nstart_time = time.time()\n\nvalid_loss_min = np.Inf     # track change in validation loss\nvalid_mae_loss_min = np.Inf # track change in validation loss\n\noutput_file_nb = 0\nmax_output_file_nb = 200\n\nfor epoch in range(1, n_epochs+1):\n\n    if (time.time() - start_time) / 3600 > 3 or (output_file_nb > max_output_file_nb):\n        output_file_nb += 1\n        print('Last iteration ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_loss_min, valid_loss))\n        torch.save(model.state_dict(), 'model_last.pt')\n        break\n    \n    # keep track of training and validation loss\n    train_loss = 0.0\n    valid_loss = 0.0\n    valid_mae_loss = 0.0\n    \n    ###################\n    # train the model #\n    ###################\n    model.train()\n    for ind, (data, target) in enumerate(train_loader):\n        print(ind, end='\\r')\n        \n        if train_on_gpu:\n            data, target = data.cuda(), target.cuda()\n        \n        optimizer.zero_grad()\n        output = model(data)\n        \n        loss = criterion(output.view(data.shape[0]), target.float())\n#         mae_score = mae(output.view(data.shape[0]), target.float())\n        \n        loss.backward()\n        optimizer.step()\n    \n        train_loss += loss.item() * data.size(0)\n        \n#         mae_train_loss +=  mae_score.item() * data.size(0)\n        \n        \n    ######################    \n    # validate the model #\n    ######################\n    model.eval()\n    for data, target in valid_loader:\n        # move tensors to GPU if CUDA is available\n        if train_on_gpu:\n            data, target = data.cuda(), target.cuda()\n\n        # forward pass: compute predicted outputs by passing inputs to the model\n        output = model(data)    \n        loss = criterion(output.view(data.shape[0]), target.float())    \n        valid_loss += loss.item() * data.size(0)\n        \n        mae_loss = mae(output.view(data.shape[0]), target.float())    \n        valid_mae_loss += mae_loss.item() * data.size(0)\n \n    # calculate average losses\n    train_loss = train_loss / len(train_loader.sampler)\n    valid_loss = valid_loss / len(valid_loader.sampler)\n    valid_mae_loss = valid_mae_loss / len(valid_loader.sampler)\n    \n    # print training/validation statistics \n    print('Epoch: {} \\tTr. Loss: {:.6f} \\tVal. Loss: {:.6f} \\tMae: {:.6f}'.format(\n        epoch, train_loss, valid_loss, valid_mae_loss))\n    \n    loss_per_iter.append([train_loss, valid_loss, valid_mae_loss])\n    \n    # save model if validation loss has decreased\n    if valid_loss <= valid_loss_min:\n        output_file_nb += 1\n        print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n        valid_loss_min,\n        valid_loss))\n        torch.save(model.state_dict(), f'model_best.pt')\n        valid_loss_min = valid_loss\n        \n        if epoch > 20:\n            output_file_nb += 1\n            torch.save(model.state_dict(), f'model_t{round(train_loss, 3)}_v{round(valid_loss, 3)}_mae{round(valid_mae_loss, 3)}_ep{epoch}.pt')\n        \n    elif valid_loss <= 1.1 * valid_loss_min and epoch > 20:\n        output_file_nb += 1\n        print('Validation loss saved at ({:.6f}).  Saving model ...'.format(valid_loss))\n        torch.save(model.state_dict(), f'model_t{round(train_loss, 3)}_v{round(valid_loss, 3)}_mae{round(valid_mae_loss, 3)}_ep{epoch}.pt')\n    \n    elif valid_mae_loss <= 1.1 * valid_mae_loss_min and epoch > 20:\n        output_file_nb += 1\n        print('Validation loss saved at ({:.6f}).  Saving model ...'.format(valid_loss))\n        torch.save(model.state_dict(), f'model_t{round(train_loss, 3)}_v{round(valid_loss, 3)}_mae{round(valid_mae_loss, 3)}_ep{epoch}.pt')\n        \n    if valid_mae_loss < valid_mae_loss_min:\n        valid_mae_loss_min = valid_mae_loss\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(14,8))\nplt.plot(np.array(loss_per_iter)[:, 0], 'o-', label = 'train')\nplt.plot(np.array(loss_per_iter)[:, 1], 'o-', label = 'valid')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(14,8))\nplt.plot(np.array(loss_per_iter)[-50:, 0], 'o-', label = 'train')\nplt.plot(np.array(loss_per_iter)[-50:, 1], 'o-', label = 'valid')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(14,8))\nplt.plot(np.array(loss_per_iter)[:, 2], 'o-', label = 'mae')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(np.array(loss_per_iter)[:, 0].min())\nprint(np.array(loss_per_iter)[:, 1].min())\nprint(np.array(loss_per_iter)[:, 2].min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}