{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Utils"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset\nfrom tqdm import tqdm_notebook as tqdm\nfrom matplotlib import pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\nimport math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\n# Early Stopping\nclass EarlyStopping:\n    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n    def __init__(self, model_dir, patience=7, verbose=False, delta=0):\n        \"\"\"\n        Args:\n            patience (int): How long to wait after last time validation loss improved.\n                            Default: 7\n            verbose (bool): If True, prints a message for each validation loss improvement. \n                            Default: False\n            delta (float): Minimum change in the monitored quantity to qualify as an improvement.\n                            Default: 0\n        \"\"\"\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n        self.model_dir = model_dir\n\n    def __call__(self, val_loss, model):\n\n        score = -val_loss\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n        elif score < self.best_score - self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model):\n        '''Saves model when validation loss decrease.'''\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), 'best_model.pt')\n        self.val_loss_min = val_loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Activation"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom torch.nn import functional\n\n\ndef shifted_softplus(x):\n    r\"\"\"Compute shifted soft-plus activation function.\n    .. math::\n       y = \\ln\\left(1 + e^{-x}\\right) - \\ln(2)\n    Args:\n        x (torch.Tensor): input tensor.\n    Returns:\n        torch.Tensor: shifted soft-plus of input.\n    \"\"\"\n    return functional.softplus(x) - np.log(2.0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Base"},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.spatial import distance_matrix\nimport torch\nfrom torch import nn\nfrom torch.nn import init\nfrom torch.nn.init import xavier_uniform_\nfrom functools import partial\nimport math\n\nfrom torch.nn.init import constant_\n\nzeros_initializer = partial(constant_, val=0.0)\n\n\nclass Dense(nn.Linear):\n    r\"\"\"Fully connected linear layer with activation function.\n    .. math::\n       y = activation(xW^T + b)\n    Args:\n        in_features (int): number of input feature :math:`x`.\n        out_features (int): number of output features :math:`y`.\n        bias (bool, optional): if False, the layer will not adapt bias :math:`b`.\n        activation (callable, optional): if None, no activation function is used.\n        weight_init (callable, optional): weight initializer from current weight.\n        bias_init (callable, optional): bias initializer from current bias.\n    \"\"\"\n\n    def __init__(\n        self, in_features,\n        out_features,\n        bias=True,\n        activation=None,\n        weight_init=xavier_uniform_,\n        bias_init=zeros_initializer,\n    ):\n        self.weight_init = weight_init\n        # self.bias_init = bias_init\n        self.activation = activation\n        # initialize linear layer y = xW^T + b\n        super(Dense, self).__init__(in_features, out_features, bias)\n\n\n    def reset_parameters(self):\n        \"\"\"Reinitialize model weight and bias values.\"\"\"\n        self.weight_init(self.weight)\n        if self.bias is not None:\n            fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)\n            bound = 1 / math.sqrt(fan_in)\n            init.uniform_(self.bias, -bound, bound)\n\n\n\n    def forward(self, inputs):\n        \"\"\"Compute layer output.\n        Args:\n            inputs (dict of torch.Tensor): batch of input values with shape B * N * F\n            mask (callable, optional): mask for remove non-exist atoms in molecular with shape B * N\n        Returns:\n            torch.Tensor: layer output.\n        \"\"\"\n        # compute linear layer y = xW^T + b\n        y = super(Dense, self).forward(inputs)\n        # add activation function\n        if self.activation:\n            y = self.activation(y)\n        return y\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Cutoff"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n    Cut off Layer: Make mask for cut off distance that higher than r cutoff\n    Param:\n        dist: distance matrix of atoms in molecular: Shape is B*N*N\n        \n    Out: \n        mask: mask of cutoff distance\n\"\"\"\n\nimport torch \nfrom torch import nn\nimport torch.nn.functional as F \nfrom torch.nn import Parameter\n\nimport numpy as np \n\nfrom scipy.spatial import distance_matrix\n\nclass Cutoff(nn.Module):\n    def __init__(self, cutoff=5.0):\n        super(Cutoff, self).__init__()\n        self.cutoff = cutoff\n    \n    def forward(self, dist):\n        \"\"\"Compute cutoff.\n        Args:\n            distances (torch.Tensor): values of interatomic distances.\n        Returns:\n            torch.Tensor: values of cutoff function.\n        \"\"\"\n        mask = (dist <= self.cutoff).float()\n        return mask \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport pickle\nfrom scipy.spatial import distance_matrix\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nfrom torch import nn \nfrom torch.nn import Parameter\nfrom torch.utils.data import Dataset, DataLoader\nimport time\natoms_z_value = {'H': 1, 'C': 12, 'O': 16, 'N': 14, 'F': 19}\n\ndef type2value(atom_type):\n    return [atoms_z_value[i] for i in atom_type]\n\nclass MoleculeData(Dataset):\n    def __init__(self, data_path, structure_path, molecules_ids, mol_path, train=True, name_feat='homo', for_Jcoupling=False):\n        self.data = pd.read_csv(data_path)\n        self.structure = pd.read_csv(structure_path)\n        self.molecules_ids = molecules_ids\n        self.for_Jcoupling = for_Jcoupling\n        self.mol_feats = pd.read_csv(mol_path)\n        self.name_feat = name_feat\n        self.group_struct = np.array(self.structure.groupby('molecule_name'))\n        self.train = train\n        start = time.time()\n        for i in range(len(self.group_struct)):\n            self.group_struct[i][1] = self.group_struct[i][1].values\n            if i % 5000 == 0:\n                print(i)\n        self.dict_struct_mol = {name: struct for name, struct in self.group_struct}\n        self.group_mol_feats = np.array(self.mol_feats.groupby('molecule_name'))\n        for i in range(len(self.group_mol_feats )):\n            self.group_mol_feats[i][1] = self.group_mol_feats[i][1].values\n            if i % 5000 == 0:\n                print(i)\n        self.dict_mol_feats = {name: feats for name, feats in self.group_mol_feats}\n        self.name_mol_feats = np.array(self.mol_feats.columns)\n        self.name_feats_to_index = {name: idx for idx, name in enumerate(self.name_mol_feats)}\n        print(' -- ex time is %s second' % (time.time() - start))\n\n\n\n    def __len__(self):\n        return len(self.molecules_ids)\n\n    def __getitem__(self, idx):\n        # struct = self.structure[self.structure['molecule_name'] == self.molecules_ids[idx]]\n        struct = self.dict_struct_mol[self.molecules_ids[idx]]\n        num_atoms = len(struct)\n        atoms_input = np.zeros((29))\n        distances = np.zeros((29, 29))\n        y_true = np.zeros((29, 29, 1))\n        mask_non_exist_node = np.zeros((29, 29))\n        xyz = struct[:, 3:]\n        atom_type = struct[:, 2]\n        atom_feat = type2value(atom_type)\n        atom_feat = np.array(atom_feat)\n        # data = self.data[self.data['molecule_name'] == self.molecules_ids[idx]]\n        # atom_index_0 = data['atom_index_0'].values.reshape(-1)\n        # atom_index_1 = data['atom_index_1'].values.reshape(-1)\n        distance = distance_matrix(xyz, xyz)\n        mask = (distance + np.eye(len(distance))) != 0\n        mask = mask.astype(np.float)\n        node_mask = np.zeros((29))\n        node_mask[np.arange(num_atoms)] = 1\n        # print(mask.shape)\n        mask_non_exist_node[:mask.shape[0], :mask.shape[1]] = mask\n        atoms_input[:atom_feat.shape[0]] = atom_feat\n        distances[:distance.shape[0], :distance.shape[1]] = distance\n        if self.train == False:\n            y_true = None\n        else:\n            if self.for_Jcoupling:\n                y_true = None\n                # y = np.zeros_like(distance)\n                # y[atom_index_0, atom_index_1] = data['scalar_coupling_constant'].values.reshape(-1)\n                # y[atom_index_1, atom_index_0] = y[atom_index_0, atom_index_1]\n                # y_true[:y.shape[0], :y.shape[1], 0] = y\n                # y_true = y_true.reshape(-1, 1)\n            else:\n                # y_true = self.mol_feats[self.mol_feats['molecule_name'] == self.molecules_ids[idx]]['homo'].values\n                y_true = self.dict_mol_feats[self.molecules_ids[idx]].reshape(-1)\n                # print(y_true)\n                idx_feat = self.name_feats_to_index[self.name_feat]\n                # print(idx_feat)\n                y_true = y_true[idx_feat]\n        \n        return atoms_input, distances, y_true, mask_non_exist_node, node_mask\n\nclass MoleculeData_v1(Dataset):\n    def __init__(self, molecules_ids, mol_path, adj_path, dict_mol_feats, dict_struct_mol, train=True, name_feat='alpha', for_Jcoupling=False):\n        self.molecules_ids = molecules_ids\n        self.for_Jcoupling = for_Jcoupling\n        self.mol_feats = pd.read_csv(mol_path)\n        self.name_feat = name_feat\n        self.train = train\n        self.dict_struct_mol = dict_struct_mol\n        self.dict_mol_feats = dict_mol_feats\n        self.name_mol_feats = np.array(self.mol_feats.columns)\n        self.name_feats_to_index = {name: idx for idx, name in enumerate(self.name_mol_feats)}\n        with open(adj_path, 'rb') as f:\n            self.adj_matrix = pickle.load(f)\n        print('DONE')\n    \n        \n\n    def __len__(self):\n        return len(self.molecules_ids)\n\n    def __getitem__(self, idx):\n        struct = self.dict_struct_mol[self.molecules_ids[idx]]\n        num_atoms = len(struct)\n        atoms_input = np.zeros((29))\n        distances = np.zeros((29, 29))\n        y_true = np.zeros((29, 29, 1))\n        # mask_non_exist_node = np.zeros((29, 29))\n        adj = self.adj_matrix[self.molecules_ids[idx]]\n        xyz = struct[:, 3:]\n        atom_type = struct[:, 2]\n        atom_feat = type2value(atom_type)\n        atom_feat = np.array(atom_feat)\n        distance = distance_matrix(xyz, xyz)\n        # mask = (distance + np.eye(len(distance))) != 0\n        # mask = mask.astype(np.float)\n        node_mask = np.zeros((29))\n        node_mask[np.arange(num_atoms)] = 1\n        # print(mask.shape)\n        # mask_non_exist_node[:mask.shape[0], :mask.shape[1]] = mask\n        atoms_input[:atom_feat.shape[0]] = atom_feat\n        distances[:distance.shape[0], :distance.shape[1]] = distance\n        if self.train == False:\n            y_true = None\n        else:\n            if self.for_Jcoupling:\n                y_true = None\n              \n            else:\n                y_true = self.dict_mol_feats[self.molecules_ids[idx]].reshape(-1)\n                idx_feat = self.name_feats_to_index[self.name_feat]\n                y_true = y_true[idx_feat]\n        \n        return atoms_input, distances, y_true, adj, node_mask\n\n\n\ndef get_train_valid_test_mol_name():\n    train_df = pd.read_csv('../input/qm9-csv/atoms_df.csv')\n    num_atoms = train_df.groupby('molecule_name').agg({\"atom_type\": 'count'}).values.reshape(-1)\n    mols_name = train_df['molecule_name'].unique()\n    mols_name_train, mols_name_valid = train_test_split(mols_name, test_size=0.15, random_state=42, stratify=num_atoms)\n    valid_df = train_df.loc[train_df['molecule_name'].isin(mols_name_valid)]\n    num_atoms_valid = valid_df.groupby('molecule_name').agg({'atom_type': 'count'}).values.reshape(-1)\n    print(num_atoms_valid)\n    mols_name_valid, mols_name_test = train_test_split(valid_df['molecule_name'].unique(), test_size=0.5, random_state=42)\n    return mols_name_train, mols_name_valid, mols_name_test\n\n\ndef gen_distence():\n    structure_df = pd.read_csv('../input/champs-scalar-coupling/structures.csv')\n    n_mols_train = len(structure_df['molecule_name'].unique())\n    distances = np.zeros((n_mols_train, 29, 29, 1))\n    for i, struct in enumerate(structure_df.groupby('molecule_name')):\n        distance = np.zeros((29, 29, 1))\n        # print(type(struct))\n        # print(struct[1])\n        xyz = struct[1][['x', 'y', 'z']].values\n        dist = distance_matrix(xyz, xyz)\n        distance[:dist.shape[0], :dist.shape[1], 0] = dist\n        distances[i] = distance\n    print(distances[0])\n    np.savez_compressed(\"distances.npz\" , distances)\n\ndef gen_type_atom():\n    structure_df = pd.read_csv('../input/champs-scalar-coupling/structures.csv')\n    n_mols_train = len(structure_df['molecule_name'].unique())\n    type_atom_list = np.zeros((n_mols_train, 29, 1))\n    for i, struct in enumerate(structure_df.groupby('molecule_name')):\n        atom = np.zeros((29, 1))\n        atom_type = struct[1]['atom'].values\n        atom_feat = type2value(atom_type)\n        atom_feat = np.array(atom_feat)\n        atom[:atom_feat.shape[0], 0] = atom_feat\n        type_atom_list[i] = atom\n    \n    print(type_atom_list[0])\n    np.savez_compressed('atom_feat.npz', type_atom_list)\n\ndef get_mol_features(data_path):\n    \"\"\"\n    Get molecular features for testing and training model\n    \"\"\"\n    data = pd.read_csv(data_path)\n    print(data.head())\n    print(data.columns)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Embedding layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n    Embedding Layer: Create embedding for atoms and edges in molecular.\n\"\"\"\n\nimport torch\nfrom torch import nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F \n\nclass EmbeddingLayer(nn.Module):\n    def __init__(self, max_z, out_dims):\n        super(EmbeddingLayer, self).__init__()\n        self.out_dims = out_dims\n        max_z = 116\n        self.embedd = nn.Embedding(max_z, out_dims, padding_idx=0)\n        # self.embedd = nn.Linear(1, out_dims, bias=False)\n\n    def forward(self, Z_value):\n        \"\"\"\n            Create embedding for Z value of atom\n        \"\"\"\n        return self.embedd(Z_value)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Radam"},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nimport torch\nfrom torch.optim.optimizer import Optimizer, required\n\nclass RAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n        self.buffer = [[None, None, None] for ind in range(10)]\n        super(RAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(RAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                buffered = self.buffer[int(state['step'] % 10)]\n                if state['step'] == buffered[0]:\n                    N_sma, step_size = buffered[1], buffered[2]\n                else:\n                    buffered[0] = state['step']\n                    beta2_t = beta2 ** state['step']\n                    N_sma_max = 2 / (1 - beta2) - 1\n                    N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n                    buffered[1] = N_sma\n\n                    # more conservative since it's an approximated value\n                    if N_sma >= 5:\n                        step_size = math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    else:\n                        step_size = 1.0 / (1 - beta1 ** state['step'])\n                    buffered[2] = step_size\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:            \n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size * group['lr'], exp_avg, denom)\n                else:\n                    p_data_fp32.add_(-step_size * group['lr'], exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n\nclass PlainRAdam(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):\n        defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)\n\n        super(PlainRAdam, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(PlainRAdam, self).__setstate__(state)\n\n    def step(self, closure=None):\n\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('RAdam does not support sparse gradients')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                state['step'] += 1\n                beta2_t = beta2 ** state['step']\n                N_sma_max = 2 / (1 - beta2) - 1\n                N_sma = N_sma_max - 2 * state['step'] * beta2_t / (1 - beta2_t)\n\n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)\n\n                # more conservative since it's an approximated value\n                if N_sma >= 5:                    \n                    step_size = group['lr'] * math.sqrt((1 - beta2_t) * (N_sma - 4) / (N_sma_max - 4) * (N_sma - 2) / N_sma * N_sma_max / (N_sma_max - 2)) / (1 - beta1 ** state['step'])\n                    denom = exp_avg_sq.sqrt().add_(group['eps'])\n                    p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n                else:\n                    step_size = group['lr'] / (1 - beta1 ** state['step'])\n                    p_data_fp32.add_(-step_size, exp_avg)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss\n\n\nclass AdamW(Optimizer):\n\n    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, warmup = 0):\n        defaults = dict(lr=lr, betas=betas, eps=eps,\n                        weight_decay=weight_decay, warmup = warmup)\n        super(AdamW, self).__init__(params, defaults)\n\n    def __setstate__(self, state):\n        super(AdamW, self).__setstate__(state)\n\n    def step(self, closure=None):\n        loss = None\n        if closure is not None:\n            loss = closure()\n\n        for group in self.param_groups:\n\n            for p in group['params']:\n                if p.grad is None:\n                    continue\n                grad = p.grad.data.float()\n                if grad.is_sparse:\n                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')\n\n                p_data_fp32 = p.data.float()\n\n                state = self.state[p]\n\n                if len(state) == 0:\n                    state['step'] = 0\n                    state['exp_avg'] = torch.zeros_like(p_data_fp32)\n                    state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)\n                else:\n                    state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)\n                    state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)\n\n                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']\n                beta1, beta2 = group['betas']\n\n                state['step'] += 1\n\n                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)\n                exp_avg.mul_(beta1).add_(1 - beta1, grad)\n\n                denom = exp_avg_sq.sqrt().add_(group['eps'])\n                bias_correction1 = 1 - beta1 ** state['step']\n                bias_correction2 = 1 - beta2 ** state['step']\n                \n                if group['warmup'] > state['step']:\n                    scheduled_lr = 1e-8 + state['step'] * group['lr'] / group['warmup']\n                else:\n                    scheduled_lr = group['lr']\n\n                step_size = scheduled_lr * math.sqrt(bias_correction2) / bias_correction1\n                \n                if group['weight_decay'] != 0:\n                    p_data_fp32.add_(-group['weight_decay'] * scheduled_lr, p_data_fp32)\n\n                p_data_fp32.addcdiv_(-step_size, exp_avg, denom)\n\n                p.data.copy_(p_data_fp32)\n\n        return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RBF "},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nimport numpy as np \n\nfrom scipy.spatial import distance_matrix\n\n\ndef gaussian_smearing(distances, offset, widths, centered=False):\n    r\"\"\"Smear interatomic distance values using Gaussian functions.\n    Args:\n        distances (torch.Tensor): interatomic distances of (N_b x N_at x N_nbh) shape.\n        offset (torch.Tensor): offsets values of Gaussian functions.\n        widths: width values of Gaussian functions.\n        centered (bool, optional): If True, Gaussians are centered at the origin and\n            the offsets are used to as their widths (used e.g. for angular functions).\n    Returns:\n        torch.Tensor: smeared distances (B x N x N x N_g).\n    \"\"\"\n    if not centered:\n        # compute width of Gaussian functions (using an overlap of 1 STDDEV)\n        coeff = -0.5 / torch.pow(widths, 2)\n        # Use advanced indexing to compute the individual components\n        diff = distances[ :, :, :, None] - offset[None, None, None, :]\n    else:\n        # if Gaussian functions are centered, use offsets to compute widths\n        coeff = -0.5 / torch.pow(offset, 2)\n        # if Gaussian functions are centered, no offset is subtracted\n        diff = distances[ :, :, None]\n    # compute smear distance values\n    gauss = torch.exp(coeff * torch.pow(diff, 2))\n    return gauss\n\n\nclass GaussianSmearing(nn.Module):\n    r\"\"\"Smear layer using a set of Gaussian functions.\n    Args:\n        start (float, optional): center of first Gaussian function, :math:`\\mu_0`.\n        stop (float, optional): center of last Gaussian function, :math:`\\mu_{N_g}`\n        n_gaussians (int, optional): total number of Gaussian functions, :math:`N_g`.\n        centered (bool, optional): If True, Gaussians are centered at the origin and\n            the offsets are used to as their widths (used e.g. for angular functions).\n        trainable (bool, optional): If True, widths and offset of Gaussian functions\n            are adjusted during training process.\n    \"\"\"\n\n    def __init__(\n        self, start=0.0, stop=5.0, n_gaussians=25, centered=False, trainable=False\n    ):\n        super(GaussianSmearing, self).__init__()\n        # compute offset and width of Gaussian functions\n        offset = torch.linspace(start, stop, n_gaussians)\n        # width is gamma parameter\n        widths = torch.FloatTensor((offset[1] - offset[0]) * torch.ones_like(offset))\n        if trainable:\n            self.width = nn.Parameter(widths)\n            self.offsets = nn.Parameter(offset)\n        else:\n            self.register_buffer(\"width\", widths)\n            self.register_buffer(\"offsets\", offset)\n        self.centered = centered\n\n    def forward(self, distances):\n        \"\"\"Compute smeared-gaussian distance values.\n        Args:\n            distances (torch.Tensor): interatomic distance values of\n                (B x N x N) shape.\n        Returns:\n            torch.Tensor: layer output of (B x N x N x N_g) shape.\n        \"\"\"\n        return gaussian_smearing(\n            distances, self.offsets, self.width, centered=self.centered\n        )\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Interaction"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F \n\n\nclass GenerateFilter(nn.Module):\n    def __init__(self, n_basis, n_filter):\n        super(GenerateFilter, self).__init__()\n        self.n_basis = n_basis\n        self.n_filter = n_filter\n        self.FC1 = Dense(self.n_basis, self.n_filter, activation=shifted_softplus)\n        self.FC2 = Dense(self.n_filter, self.n_filter)\n\n    def forward(self, rbf_filer, cutoff_mask):\n        \"\"\"\n            Generate continuous filter\n            Params:\n                rbf_filter: rbf_filter that was generated in RBF layer.\n                    Shape B * N * N * n_basis\n                cutoff_mask: cutoff mask for remove distance higher than r cut-off\n                    Shape N * N\n            Out:\n                filter: Shape B * N * N * n_filter\n        \"\"\"\n        gen_filter = self.FC1(rbf_filer)\n        gen_filter = self.FC2(gen_filter)\n        gen_filter = gen_filter * cutoff_mask.unsqueeze(-1)\n      \n        return gen_filter\n\nclass CFConv(nn.Module):\n    def __init__(self, n_filter, in_atom_feat, out_atom_feat):\n        super(CFConv, self).__init__()\n        self.n_filter = n_filter\n        self.in_atom_feat = in_atom_feat\n        self.out_atom_feat = out_atom_feat\n        self.in2f = Dense(self.in_atom_feat, self.n_filter, activation=None)\n        self.f2out = Dense(self.n_filter, self.out_atom_feat, activation=shifted_softplus)\n\n        \n\n    def forward(self, x, rbf_filter, mask):\n        \"\"\"\n            CFConv for update atom features\n            Params:\n                x: atom features with shape B * N*in_atom_feat\n                rbf_filter: filter with shape B * N*N*n_filter\n                mask: remove non exist node, shape B * N * N\n            Out:\n                new_x: update atom features with shape N*out_atom_feat\n            Update in future: \n                If want to aggregate message from neighborhood node, using mask neighborhood as param and \n                element wise multiplication it with filter.\n        \"\"\"\n        n_atoms = x.shape[1]\n        batch_size = x.shape[0]\n        x_ = self.in2f(x)\n        # reshape x_ to n_atom*n_atom_n_filter for element wise multiplication\n        x_ = x_.repeat(1, n_atoms, 1).reshape(batch_size,n_atoms, n_atoms, -1)\n\n        # filter\n        x_ = x_ * rbf_filter\n        # remove non exist node\n        x_ = x_ * mask.unsqueeze(-1)\n\n        # aggregator message, just sum message from all node in the molecular, future we can aggregator message from neighborhood by using adjacency matrix mask\n        x_ = x_.sum(2)\n        x_new = self.f2out(x_)\n        # x_new = self.f2out_2(x_new)\n        return x_new\n\n\nclass InteractionLayer(nn.Module):\n    def __init__(self, n_filter, in_atom_feat, out_atom_feat, n_gaussians=200, cutoff=5.0):\n        super(InteractionLayer, self).__init__()\n        self.n_filter = n_filter\n        self.in_atom_feat = in_atom_feat\n        self.out_atom_feat = out_atom_feat\n        self.gen_filter = GenerateFilter(n_gaussians, self.n_filter)\n        self.cfconv = CFConv( self.n_filter, self.in_atom_feat, self.out_atom_feat)\n        self.out_layer = Dense(out_atom_feat, out_atom_feat, activation=None, bias=True)\n        self.RBF_expansion = GaussianSmearing(start=0.0, stop=20.0, n_gaussians=n_gaussians)\n        self.Cutoff = Cutoff(cutoff)\n\n    def forward(self, x, distance_matrix, mask):\n        \"\"\"\n            Interaction Layer: compute interaction of atom wise in molecular.\n            Params:\n                x: atom features with shape B*N*in_atom_feat\n                distance matrix: distance matrix of atoms in molecular. \n                    Shape is B * N * N\n                mask: remove non exist node\n                    Shape is B * N * N\n            Out: \n                x_new: new atom features with shape B * N * out_atom_feat\n            Here we have n_basis is equal n_gaussians and both are the number of filter the generate in RBF expanssion\n        \"\"\"\n        # print('distance matrix is ', distance_matrix, '\\n with shape is ', distance_matrix.shape )\n        rbf_filter = self.RBF_expansion(distance_matrix)\n        # print('rbf filter is ', rbf_filter, '\\n with shape is ', rbf_filter.shape)\n        cutoff_mask = self.Cutoff(distance_matrix)\n        # print('cut of mask is ', cutoff_mask, '\\n with shape is ', cutoff_mask.shape)\n        filter_ = self.gen_filter(rbf_filter, cutoff_mask)\n        # print('filter is ', filter_, '\\n with shape is ', filter_.shape)\n        x_new = self.cfconv(x, filter_, mask)\n        # print('x new after through cfconv is ', x_new, '\\nwith shape is ', x_new.shape)\n        x_new = self.out_layer(x_new)\n        # print(\"out \", x_new)\n\n        return x_new\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Readout"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn \nfrom torch.nn import Parameter\nimport torch.nn.functional as F \n\nclass Readout_basis(nn.Module):\n    def __init__(self, in_atom_feat, state_dims):\n        super(Readout_basis, self).__init__()\n        self.FC1 = Dense(in_atom_feat, in_atom_feat, shifted_softplus)\n        self.FC2 = Dense(in_atom_feat, state_dims, shifted_softplus)\n        self.FC3 = Dense(state_dims, state_dims, shifted_softplus)\n        self.FC4 = Dense(2*state_dims, state_dims)\n        self.out_layer = Dense(state_dims, 1, shifted_softplus)\n\n    def forward(self, x):\n        \"\"\"\n            Calculate J coupling of atom pair in the molecular\n            Params:\n                x: atom features with shape N * in_atom_feat\n            Out:\n                J coupling with shape N^2 * 1\n        \"\"\"\n        N = x.shape[1]\n        B = x.shape[0]\n        x_new = self.FC1(x)\n        x_new = self.FC2(x_new)\n        x_new = self.FC3(x_new)\n        pair = torch.cat([x_new.repeat(1, 1, N).view(B, N*N, -1), x_new.repeat(1, N, 1)], dim=-1)\n        pair = self.FC4(pair)\n        out = self.out_layer(pair)\n        return out\n    \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Schnet"},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F \n\n\nclass Schnet_basic(nn.Module):\n    def __init__(self, n_atom_basis, n_interactions=3, cutoff=5.0, n_gaussians=200, max_z=100, n_filter=128):\n        super(Schnet_basic, self).__init__()\n        self.embedd_layer = EmbeddingLayer(max_z, n_atom_basis)\n        self.interactions = nn.ModuleList([InteractionLayer(n_filter, n_atom_basis, n_atom_basis, n_gaussians) for _ in range(n_interactions)])\n        # self.readout = Readout_basis(n_atom_basis,n_atom_basis)\n        self.atom_wise_64 = Dense(n_atom_basis, 64, activation=shifted_softplus)\n        self.atom_wise_32_1 = Dense(64, 32, activation=shifted_softplus)\n\n        self.atom_wise_1 = Dense(32, 1, activation=None)\n\n    def forward(self, z, distance_matrix, mask, node_mask):\n        \"\"\"\n            Params: \n                z: nuclear charge of atoms\n                distance_matrix: distances matrix between atoms in molecular\n            Out:\n                J coupling between atom pairs\n        \"\"\"                \n        x = self.embedd_layer(z)\n        # print(x.shape)\n        # print(distance_matrix.shape)\n        # mask = distance_matrix != 0\n        # mask = mask.float() + torch.eye(29).unsqueeze(0).cuda()\n        # print('begin, node feats x is ', x, ' \\nshape of x is ', x.shape)\n        for interacton in self.interactions:\n            v = interacton(x, distance_matrix, mask)\n            x = x + v\n            # print('node feats after interaction layer is ', x, '\\n shape of x is ', x.shape)\n\n        # out = self.readout(x)\n        x = self.atom_wise_64(x)\n        x = self.atom_wise_32_1(x)\n        x = self.atom_wise_1(x)\n        x = x * node_mask.unsqueeze(-1)\n        x = x.squeeze()\n        out = x.sum(-1)\n\n        return out\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# !mkdir '..working/model/version_1'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division\nfrom __future__ import print_function\n\nimport os\nimport glob\nimport time\nimport random\nimport argparse\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader\nfrom torch.autograd import Variable\nimport tqdm\nfrom tqdm import tqdm_notebook as tqdm\n\nimport torch.multiprocessing\ntorch.multiprocessing.set_sharing_strategy('file_system')\nimport random \nimport pandas as pd\nimport pickle\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(device)\nprint(torch.cuda.is_available())\n\nmodel_dir = './model/version_1/'\n\ndef log_mae(out, label):\n    out = out.view(-1)\n    label = label.view(-1)\n    loss = torch.abs(out - label)\n    loss = loss.mean()\n    loss = torch.log(loss)\n    return loss\n\ndef mae(out, label):\n    out = out.view(-1)\n    label = label.view(-1)\n    loss = torch.abs(out - label)\n    loss = loss.mean()\n    return loss\n\ndef count_parameters(model):\n    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n\ntrain_data_path = '../input/champs-scalar-coupling/train.csv'\nstructure_path = '../input/champs-scalar-coupling/structures.csv'\nmol_path = '../input/qm9-csv/stats_df_paper_used.csv'\nadj_path = '../input/qm9-extradata/adj_matrix_with_align.pkl'\n# train_atom_feats_path = 'trainatom_feat.npz'\n# train_distances_path = 'traindistances.npz'\n# train_mask_none_exist_path = 'trainmask_none_exist.npz'\n# train_node_mask_path = 'trainnode_mask.npz'\n\n# test_atom_feats_path = 'testatom_feat.npz'\n# test_distances_path = 'testdistances.npz'\n# test_mask_none_exist_path = 'testmask_none_exist.npz'\n# test_node_mask_path = 'testnode_mask.npz'\n\n\nstructure = pd.read_csv(structure_path)\nmol_feats = pd.read_csv(mol_path)\n\n\n# name_feat = 'U0'\n# scale_max = mol_feats[name_feat].max()\n# scale_min = mol_feats[name_feat].min()\n# scale_norm = scale_max - scale_min\n# scale_mid = (scale_max + scale_min) / 2\n# mol_feats[name_feat + \"no_norm\"] = mol_feats[name_feat]\n# mol_feats[name_feat] = (mol_feats[name_feat] - scale_mid) / scale_norm\n\n# print(scale_mid, scale_norm)\n# print(mol_feats.head())\n\ngroup_struct = np.array(structure.groupby('molecule_name'))\nstart = time.time()\nfor i in range(len(group_struct)):\n    group_struct[i][1] = group_struct[i][1].values\n    if i % 5000 == 0:\n        print(i)\ndict_struct_mol = {name: struct for name, struct in group_struct}\ngroup_mol_feats = np.array(mol_feats.groupby('molecule_name'))\nfor i in range(len(group_mol_feats )):\n    group_mol_feats[i][1] = group_mol_feats[i][1].values\n    if i % 5000 == 0:\n        print(i)\ndict_mol_feats = {name: feats for name, feats in group_mol_feats}\nprint(' -- ex time is %s second' % (time.time() - start))\n\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mols_name_train, mols_name_valid, mols_name_test  = get_train_valid_test_mol_name()\nprint(len(mols_name_train), len(mols_name_valid), len(mols_name_test))\n# dataset_train = MoleculeData(train_data_path, structure_path, mols_name_train, mol_path)\n# dataset_valid = MoleculeData(train_data_path, structure_path, mols_name_valid, mol_path)\ndataset_train = MoleculeData_v1(mols_name_train, mol_path, adj_path, dict_mol_feats, dict_struct_mol)\ndataset_valid = MoleculeData_v1(mols_name_valid, mol_path, adj_path, dict_mol_feats, dict_struct_mol)\ndataset_test = MoleculeData_v1(mols_name_test, mol_path, adj_path, dict_mol_feats, dict_struct_mol)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataloader_train = DataLoader(dataset_train, batch_size=32, num_workers=10, shuffle=False)\ndataloader_valid = DataLoader(dataset_valid, batch_size=32, shuffle=False, num_workers=10)\ndataloader_test = DataLoader(dataset_test, batch_size=32, shuffle=False, num_workers=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.optim.lr_scheduler import *\nfrom torch.utils.tensorboard import SummaryWriter\nwriter = SummaryWriter()\n\ndef train(n_epoch, patience=5):\n    # Model \n    model = Schnet_basic(n_atom_basis=128, n_interactions=6, cutoff=20.0, n_gaussians=200, max_z=100, n_filter=128)\n    model = model.to(device)\n\n    print(model)\n    for name, param in model.named_parameters():\n        if param.requires_grad:\n            print (name, param.data)\n    print(f'The model has {count_parameters(model):,} trainable parameters')\n\n    # Optimizer\n    optimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=5e-4)\n    criterion = mae\n\n    train_losses = []\n    valid_losses = []\n    avg_train_losses = []\n    avg_valid_losses = []\n    train_lost_iter = []\n    valid_lost_iter = []\n    lr_iter = []\n    lr_epoch = []\n    predict = []\n\n    early_stopping = EarlyStopping(model_dir=model_dir, patience=patience, verbose=True)\n#     scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1)\n#     scheduler = ReduceLROnPlateau(optimizer, 'min', 0.1, 8)\n#     scheduler = StepLR(optimizer, step_size=25, gamma=0.8)\n    iters = len(dataloader_train)\n\n    for epoch in range(1, n_epoch + 1):\n        model.train()\n        for step, (atom_feat, distance, y, mask, node_mask) in enumerate(dataloader_train):\n#             scheduler.step(epoch + i / iters)\n#             lr_iter.append(scheduler.get_lr())\n            optimizer.zero_grad()\n\n            atom_feat = atom_feat.to(device, dtype=torch.long)\n            distance = distance.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.float)\n            mask = mask.to(device, dtype=torch.float)\n            node_mask = node_mask.to(device, dtype=torch.float)\n\n            atom_feat = Variable(atom_feat)\n            distance = Variable(distance)\n            y = Variable(y)\n            mask = Variable(mask)\n            node_mask = Variable(node_mask)\n\n            y_predict = model(atom_feat, distance, mask, node_mask)\n            loss = criterion(y_predict, y)\n            loss.backward()\n            optimizer.step()\n            train_losses.append(loss.item())\n            train_lost_iter.append(loss.item())\n#             print(y, y_predict)\n            if step % 200 == 0:\n                print('y predict shape: ', y_predict.shape)\n                print('step: ', step)\n                print('loss train: ', np.array(train_losses).mean())\n                print(\"y: \", y[0].reshape(-1))\n                print(\"y predict: \", y_predict[0].reshape(-1))\n\n        model.eval()\n        for step, (atom_feat, distance, y, mask, node_mask) in enumerate(dataloader_valid):\n            atom_feat = atom_feat.to(device, dtype=torch.long)\n            distance = distance.to(device, dtype=torch.float)\n            y = y.to(device, dtype=torch.float)\n            mask = mask.to(device, dtype=torch.float)\n            node_mask = node_mask.to(device, dtype=torch.float)\n\n            atom_feat = Variable(atom_feat)\n            distance = Variable(distance)\n            y = Variable(y)\n            mask = Variable(mask)\n            node_mask = Variable(node_mask)\n\n            y_predict = model(atom_feat, distance, mask, node_mask)\n            loss = criterion(y_predict, y)\n            valid_losses.append(loss.item())\n            valid_lost_iter.append(loss.item())\n#             predict.append(y_predict)\n        \n#         scheduler.step()\n        \n        # print training/validation statistics \n        # calculate average loss over an epoch\n        train_loss = np.average(train_losses)\n        valid_loss = np.average(valid_losses)\n        avg_train_losses.append(train_loss)\n        avg_valid_losses.append(valid_loss)\n\n#         lr_epoch.append(scheduler.get_lr())\n\n        epoch_len = len(str(n_epoch))        \n        print_msg = (f'[{epoch:>{epoch_len}}/{n_epoch:>{epoch_len}}] ' +\n                     f'train_loss: {train_loss:.5f} ' +\n                     f'valid_loss: {valid_loss:.5f}')\n        print(print_msg)\n        # clear lists to track next epoch\n        train_losses = []\n        valid_losses = []\n        \n#         torch.save(model.state_dict(), model_dir+'mode_' + str(epoch) + '_.pth')\n#         torch.save(optimizer.state_dict(), model_dir+'opti_' + str(epoch) + '_.pth')\n        \n        # early_stopping needs the validation loss to check if it has decresed, \n        # and if it has, it will make a checkpoint of the current model\n        early_stopping(valid_loss, model)        \n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            # torch.save(optimizer.state_dict(), 'optimizer_of_bestmodel.pt')\n            print(epoch)\n            break\n    \n    return avg_train_losses, avg_valid_losses, train_lost_iter, valid_lost_iter, lr_iter, lr_epoch\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\navg_train_losses, avg_valid_losses, train_lost_iter, valid_lost_iter, lr_iter, lr_epoch = train(n_epoch=30, patience=8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(avg_train_losses)\nplt.plot(avg_valid_losses)\n\nplt.xlabel(\"Number of iteration\")\nplt.ylabel(\"Loss\")\nplt.legend(['train', 'valid'], loc='upper right')\nplt.title(\"Schnet: Loss vs Number of epoch\")\nplt.show()\n\n\nplt.figure(figsize=(15,15))\nplt.plot(train_lost_iter)\nplt.plot(valid_lost_iter)\nplt.xlabel(\"Schnet: Loss vs Number of iterator\")\nplt.ylabel(\"Loss\")\nplt.legend(['train', 'valid'], loc='upper left')\nplt.title(\"Schnet: Loss vs Number of iteration\")\nplt.show()\n\nplt.figure(figsize=(15,15))\nplt.plot(lr_epoch)\nplt.xlabel(\"Schnet: lr vs Number of epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Schnet: lr vs Number of epoch\")\nplt.show()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}