{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --user stg \n!pip install pytorch-tabnet wget\n!conda install -y -c rdkit rdkit\n!pip install torchviz\n!pip install -U deepchem\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\nfrom lightgbm import LGBMClassifier,LGBMRegressor\nfrom sklearn.preprocessing import QuantileTransformer\nfrom tqdm import tqdm\nimport warnings\nimport lightgbm as lgb\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import PowerTransformer,MinMaxScaler\nfrom torchviz import make_dot\n\nimport tempfile\nimport numpy as np\nimport itertools\nfrom rdkit import Chem\nfrom rdkit import DataStructs\nfrom rdkit.Chem import AllChem\nfrom rdkit.ML.Cluster import Butina\nimport deepchem as dc\nfrom typing import Dict, List, Set, Tuple, Union\nimport numpy as np\nimport pandas as pd\nfrom collections import Counter\nfrom lightgbm import LGBMClassifier,LGBMRegressor\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\nfrom sklearn.model_selection import StratifiedGroupKFold,GroupKFold,KFold,StratifiedKFold\nimport lightgbm as lgb\nfrom sklearn import preprocessing\nimport warnings\nimport time\nwarnings.filterwarnings(\"ignore\")\nimport lightgbm as lgb\nfrom bayes_opt import BayesianOptimization\nfrom stg import STG\nimport torch\nimport numpy as np\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.autograd import Variable\nfrom torch.nn.modules.loss import _WeightedLoss\nfrom sklearn.model_selection import train_test_split\nfrom keras.layers import Input, Dense, Flatten, Concatenate, concatenate, Dropout, Lambda\nfrom keras.models import Model\nfrom keras.layers.embeddings import Embedding\nfrom tqdm import tqdm\nimport tensorflow as tf\nfrom keras import backend as K\nimport numpy as np\nfrom keras.activations import softmax\nfrom pytorch_tabnet.tab_model import TabNetClassifier\nimport os\nimport wget\nfrom pathlib import Path\nimport shutil\nimport gzip\nfrom matplotlib import pyplot as plt\nfrom sklearn.cluster import KMeans\nimport random\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import roc_curve, auc\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import label_binarize\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import roc_auc_score\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n# from numba import cuda \n# device = cuda.get_current_device()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-27T16:59:38.917148Z","iopub.execute_input":"2022-06-27T16:59:38.917963Z","iopub.status.idle":"2022-06-27T17:02:10.749617Z","shell.execute_reply.started":"2022-06-27T16:59:38.917872Z","shell.execute_reply":"2022-06-27T17:02:10.748560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cross Validation Strategy\n### split the data by scaffold using generate_scaffold","metadata":{}},{"cell_type":"code","source":"class ScaffoldGenerator(object):\n  def __init__(self, include_chirality=False):\n    self.include_chirality = include_chirality\n\n  def get_scaffold(self, mol):\n    from rdkit.Chem.Scaffolds import MurckoScaffold\n    return MurckoScaffold.MurckoScaffoldSmiles(\n        mol=mol, includeChirality=self.include_chirality)\n\n\ndef generate_scaffold(smiles, include_chirality=False):\n  \"\"\"Compute the Bemis-Murcko scaffold for a SMILES string.\"\"\"\n  mol = Chem.MolFromSmiles(smiles)\n  engine = ScaffoldGenerator(include_chirality=include_chirality)\n  scaffold = engine.get_scaffold(mol)\n  return scaffold","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:02:10.751965Z","iopub.execute_input":"2022-06-27T17:02:10.753634Z","iopub.status.idle":"2022-06-27T17:02:10.761791Z","shell.execute_reply.started":"2022-06-27T17:02:10.753594Z","shell.execute_reply":"2022-06-27T17:02:10.760976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=42):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=42)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:02:10.762990Z","iopub.execute_input":"2022-06-27T17:02:10.763621Z","iopub.status.idle":"2022-06-27T17:02:10.784591Z","shell.execute_reply.started":"2022-06-27T17:02:10.763586Z","shell.execute_reply":"2022-06-27T17:02:10.783594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch Utils","metadata":{}},{"cell_type":"markdown","source":"## SmoothCrossEntropyLoss for imbalnce data","metadata":{}},{"cell_type":"code","source":"class LabelSmoothingLoss(nn.Module):\n    def __init__(self, classes, smoothing=0.0, dim=-1, weight = None):\n        \"\"\"if smoothing == 0, it's one-hot method\n           if 0 < smoothing < 1, it's smooth method\n        \"\"\"\n        super(LabelSmoothingLoss, self).__init__()\n        self.confidence = 1.0 - smoothing\n        self.smoothing = smoothing\n        self.weight = weight\n        self.cls = classes\n        self.dim = dim\n\n    def forward(self, pred, target):\n        assert 0 <= self.smoothing < 1\n        pred = pred.log_softmax(dim=self.dim)\n\n        if self.weight is not None:\n            pred = pred * self.weight.unsqueeze(0)   \n\n        with torch.no_grad():\n            true_dist = torch.zeros_like(pred)\n            true_dist.fill_(self.smoothing / (self.cls - 1))\n            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))\n    \nclass SmoothCrossEntropyLoss(_WeightedLoss):\n    def __init__(self, weight=None, reduction='mean', smoothing=0.0):\n        super().__init__(weight=weight, reduction=reduction)\n        self.smoothing = smoothing\n        self.weight = weight\n        self.reduction = reduction\n\n    def k_one_hot(self, targets:torch.Tensor, n_classes:int, smoothing=0.0):\n        with torch.no_grad():\n            targets = torch.empty(size=(targets.size(0), n_classes),\n                                  device=targets.device) \\\n                                  .fill_(smoothing /(n_classes-1)) \\\n                                  .scatter_(1, targets.data.unsqueeze(1), 1.-smoothing)\n        return targets\n\n    def reduce_loss(self, loss):\n        return loss.mean() if self.reduction == 'mean' else loss.sum() \\\n        if self.reduction == 'sum' else loss\n\n    def forward(self, inputs, targets):\n        assert 0 <= self.smoothing < 1\n\n        targets = self.k_one_hot(targets, inputs.size(-1), self.smoothing)\n        log_preds = F.log_softmax(inputs, -1)\n\n        if self.weight is not None:\n            log_preds = log_preds * self.weight.unsqueeze(0)\n        return self.reduce_loss(-(targets * log_preds).sum(dim=-1))","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:02:10.789360Z","iopub.execute_input":"2022-06-27T17:02:10.789623Z","iopub.status.idle":"2022-06-27T17:02:10.807313Z","shell.execute_reply.started":"2022-06-27T17:02:10.789600Z","shell.execute_reply":"2022-06-27T17:02:10.806329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# keras utils","metadata":{}},{"cell_type":"code","source":"def weighted_categorical_crossentropy(weights): \n    weights = K.variable(weights)\n    def loss(y_true, y_pred):\n        # scale predictions so that the class probas of each sample sum to 1\n        y_pred /= K.sum(y_pred, axis=-1, keepdims=True)\n        # clip to prevent NaN's and Inf's\n        y_pred = K.clip(y_pred, K.epsilon(), 1 - K.epsilon())\n        # calc\n        loss = y_true * K.log(y_pred) * weights\n        loss = -K.sum(loss, -1)\n        return loss\n    return loss","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:02:10.808817Z","iopub.execute_input":"2022-06-27T17:02:10.809341Z","iopub.status.idle":"2022-06-27T17:02:10.818525Z","shell.execute_reply.started":"2022-06-27T17:02:10.809303Z","shell.execute_reply":"2022-06-27T17:02:10.817461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch trainner utils","metadata":{}},{"cell_type":"markdown","source":"## Dataloder, train loop, val loop, test loop, nn model, cnn model\n### Notice the layer that adds Gaussian noise to the training, significantly improved the score","metadata":{}},{"cell_type":"code","source":"class BGUDataset:\n    def __init__(self, features, targets):\n        self.features = features\n        self.targets = targets\n        \n    def __len__(self):\n        return (self.features.shape[0])\n    \n    def __getitem__(self, idx):\n        dct = {\n            'x' : torch.tensor(self.features[idx, :], dtype=torch.float),\n            'y' : torch.tensor(self.targets[idx], dtype=torch.long)            \n        }\n        return dct\n    \nclass TestDataset:\n    def __init__(self, features):\n        self.features = features\n        \n    def __len__(self):\n        return (self.features.shape[0])\n    \n    def __getitem__(self, idx):\n        dct = {\n            'x' : torch.tensor(self.features[idx, :], dtype=torch.float)\n        }\n        return dct\n    \ndef train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):\n    model.train()\n    final_loss = 0\n    \n    for data in dataloader:\n        optimizer.zero_grad()\n        inputs, targets = data['x'].to(device), data['y'].to(device)\n        outputs = model(inputs)\n        loss = loss_fn(outputs, targets)\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n        \n        final_loss += loss.item()\n        \n    final_loss /= len(dataloader)\n    \n    return final_loss\n\n\ndef valid_fn(model, loss_fn, dataloader, device):\n    model.eval()\n    final_loss = 0\n    valid_preds = []\n    T = []\n    for data in dataloader:\n        inputs, targets = data['x'].to(device), data['y'].to(device)\n        with torch.no_grad():\n            outputs = model(inputs)\n            loss = loss_fn(outputs, targets)\n        \n        final_loss += loss.item()\n        valid_preds.append(outputs.softmax(1).detach().cpu().numpy())\n        T.append(data['y'].detach().cpu().numpy())\n    final_loss /= len(dataloader)\n    valid_preds = np.concatenate(valid_preds)\n    T = np.concatenate(T)\n    \n    \n    return final_loss, valid_preds,T\n\ndef inference_fn(model, dataloader, device):\n    model.eval()\n    preds = []\n    \n    for data in dataloader:\n        inputs = data['x'].to(device)\n\n        with torch.no_grad():\n            outputs = model(inputs)\n        \n        preds.append(outputs.softmax(1).detach().cpu().numpy())\n        \n    preds = np.concatenate(preds)\n    \n    return preds\n\nclass GaussianNoise(nn.Module):\n    def __init__(self, stddev):\n        super().__init__()\n        self.stddev = stddev\n\n    def forward(self, din):\n        if self.training:\n            return din + torch.autograd.Variable(torch.randn(din.size()).cuda() * self.stddev)\n        return din\n\nclass Model(nn.Module):\n    def __init__(self, num_features, num_targets, hidden_size):\n        super(Model, self).__init__()\n        self.batch_norm1 = nn.BatchNorm1d(num_features)\n        self.dropout1 = nn.Dropout(0.5)\n        self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))\n        \n        self.GaussianNoise = GaussianNoise(stddev=0.075)\n        \n        self.batch_norm2 = nn.BatchNorm1d(hidden_size)\n        self.dropout2 = nn.Dropout(0.5)\n        self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))\n        \n        self.batch_norm3 = nn.BatchNorm1d(hidden_size)\n        self.dropout3 = nn.Dropout(0.5)\n        self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))\n    \n    def forward(self, x):\n        x = self.batch_norm1(x)\n#         x = self.GaussianNoise(x)\n        x = self.dropout1(x)\n        x = F.relu(self.dense1(x))\n        x = self.GaussianNoise(x)\n        \n#         x = self.batch_norm2(x)\n#         x = self.dropout2(x)\n#         x = F.relu(self.dense2(x))\n        \n        x = self.batch_norm3(x)\n        x = self.dropout3(x)\n        x = self.dense3(x)\n        \n        return x\n    \n\n    \nclass ModelCNN(nn.Module):\n    def __init__(self, num_features, num_targets, hidden_size):\n            super(ModelCNN, self).__init__()\n            cha_1 = 8\n            cha_2 = 16\n            cha_3 = 16\n\n            cha_1_reshape = int(hidden_size/cha_1)\n            cha_po_1 = int(hidden_size/cha_1/2)\n            cha_po_2 = int(hidden_size/cha_1/2/2) * cha_3\n\n            self.cha_1 = cha_1\n            self.cha_2 = cha_2\n            self.cha_3 = cha_3\n            self.cha_1_reshape = cha_1_reshape\n            self.cha_po_1 = cha_po_1\n            self.cha_po_2 = cha_po_2\n\n            self.batch_norm1 = nn.BatchNorm1d(num_features)\n            self.dropout1 = nn.Dropout(0.5)\n            self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))\n\n            self.batch_norm_c1 = nn.BatchNorm1d(cha_1)\n            self.dropout_c1 = nn.Dropout(0.5)\n            self.conv1 = nn.utils.weight_norm(nn.Conv1d(cha_1,cha_2, kernel_size = 5, stride = 1, padding=2,  bias=False),dim=None)\n\n            self.ave_po_c1 = nn.AdaptiveAvgPool1d(output_size = cha_po_1)\n\n            self.batch_norm_c2 = nn.BatchNorm1d(cha_2)\n            self.dropout_c2 = nn.Dropout(0.5)\n            self.conv2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.batch_norm_c2_1 = nn.BatchNorm1d(cha_2)\n            self.dropout_c2_1 = nn.Dropout(0.5)\n            self.conv2_1 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_2, kernel_size = 3, stride = 1, padding=1, bias=True),dim=None)\n\n            self.batch_norm_c2_2 = nn.BatchNorm1d(cha_2)\n            self.dropout_c2_2 = nn.Dropout(0.5)\n            self.conv2_2 = nn.utils.weight_norm(nn.Conv1d(cha_2,cha_3, kernel_size = 5, stride = 1, padding=2, bias=True),dim=None)\n\n            self.max_po_c2 = nn.MaxPool1d(kernel_size=4, stride=2, padding=1)\n\n            self.flt = nn.Flatten()\n\n            self.batch_norm3 = nn.BatchNorm1d(cha_po_2)\n            self.dropout3 = nn.Dropout(0.2)\n            self.dense3 = nn.utils.weight_norm(nn.Linear(cha_po_2, num_targets))\n            self.GaussianNoise = GaussianNoise(stddev=0.075)\n\n    def forward(self, x):\n\n            x = self.batch_norm1(x)\n            x = self.dropout1(x)\n            x = F.celu(self.dense1(x), alpha=0.06)\n            x = self.GaussianNoise(x)\n\n            x = x.reshape(x.shape[0],self.cha_1,\n                          self.cha_1_reshape)\n\n            x = self.batch_norm_c1(x)\n            x = self.dropout_c1(x)\n            x = F.relu(self.conv1(x))\n\n            x = self.ave_po_c1(x)\n\n            x = self.batch_norm_c2(x)\n            x = self.dropout_c2(x)\n            x = F.relu(self.conv2(x))\n            x_s = x\n\n            x = self.batch_norm_c2_1(x)\n            x = self.dropout_c2_1(x)\n            x = F.relu(self.conv2_1(x))\n\n            x = self.batch_norm_c2_2(x)\n            x = self.dropout_c2_2(x)\n            x = F.relu(self.conv2_2(x))\n            x =  x * x_s\n\n            x = self.max_po_c2(x)\n\n            x = self.flt(x)\n\n            x = self.batch_norm3(x)\n            x = self.dropout3(x)\n            x = self.dense3(x)\n\n            return x","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:04:14.537425Z","iopub.execute_input":"2022-06-27T17:04:14.537955Z","iopub.status.idle":"2022-06-27T17:04:14.580520Z","shell.execute_reply.started":"2022-06-27T17:04:14.537918Z","shell.execute_reply":"2022-06-27T17:04:14.579461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=[20,20])\nplt.imshow(plt.imread(\"../input/bgudocs/exp2pipeline.png\"))\nplt.title(\"pipeline\")\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:21:24.812574Z","iopub.execute_input":"2022-06-27T17:21:24.813029Z","iopub.status.idle":"2022-06-27T17:21:25.184057Z","shell.execute_reply.started":"2022-06-27T17:21:24.812989Z","shell.execute_reply":"2022-06-27T17:21:25.183140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# preprocessing","metadata":{}},{"cell_type":"markdown","source":"## I created a new features in 4 categories using MakeData class:\n* PCA - reduced the dimension of the features from 1000+ to 600 (MakeData.pca function)\n* cluster (Kmeans) - used the numerical features and performed clustering (MakeData.fe_cluster function)\n* Statistical features - calculation of statistical indices (MakeData.fe_stats function)\n* Raw data.\n\nAfter this I connected them all together and enabled normalization of each variable between 0 and 1  (MakeData.MinMax function)\n\nAt the end I deleted features that are correlated to each other.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import PowerTransformer,MinMaxScaler\nfrom sklearn.feature_selection import VarianceThreshold\n\nclass MakeData:\n    def __init__(self,read_fast=True):\n        self.do_read_fast = read_fast\n        \n    def fe_stats(self,train, test):\n        features_c = [col for col in train if  train[col].nunique()==2 and col.startswith('X')]\n        features_g = [col for col in train if  train[col].nunique()!=2 and col.startswith('X')]\n\n        for df in train, test:\n            df['g_sum'] = df[features_g].sum(axis = 1)\n            df['g_mean'] = df[features_g].mean(axis = 1)\n            df['g_std'] = df[features_g].std(axis = 1)\n            df['g_kurt'] = df[features_g].kurtosis(axis = 1)\n            df['g_skew'] = df[features_g].skew(axis = 1)\n            df['c_sum'] = df[features_c].sum(axis = 1)\n            df['c_mean'] = df[features_c].mean(axis = 1)\n            df['c_std'] = df[features_c].std(axis = 1)\n            df['c_kurt'] = df[features_c].kurtosis(axis = 1)\n            df['c_skew'] = df[features_c].skew(axis = 1)\n            df['gc_sum'] = df[features_g + features_c].sum(axis = 1)\n            df['gc_mean'] = df[features_g + features_c].mean(axis = 1)\n            df['gc_std'] = df[features_g + features_c].std(axis = 1)\n            df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)\n            df['gc_skew'] = df[features_g + features_c].skew(axis = 1)\n        return train, test\n    \n    def fe_cluster(self,train, test, n_clusters_g = 5, n_clusters_c = 35, SEED = 123):\n    \n        features_c = [col for col in train if  train[col].nunique()==2 and col.startswith('X')]\n        features_g = [col for col in train if  train[col].nunique()!=2 and col.startswith('X')]\n\n        def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):\n            train_ = train[features].copy()\n            test_ = test[features].copy()\n            data = pd.concat([train_, test_], axis = 0)\n            kmeans = KMeans(n_clusters = n_clusters, random_state = SEED).fit(data)\n            train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]\n            test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]\n            train = pd.get_dummies(train, columns = [f'clusters_{kind}'])\n            test = pd.get_dummies(test, columns = [f'clusters_{kind}'])\n            return train, test\n\n        train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)\n        train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)\n#         train,test = self.MinMax(train,test)\n        return train, test\n    \n    def pca(self,train, test):\n        n_comp = 600\n        \n\n        data = pd.concat([pd.DataFrame(train), pd.DataFrame(test)])\n        data2 = (PCA(n_components=n_comp, random_state=42).fit_transform(data))\n        train2 = data2[:train.shape[0]]; test2 = data2[-test.shape[0]:]\n\n        train2 = pd.DataFrame(train2, columns=[f'pca-{i}' for i in range(n_comp)])\n        test2 = pd.DataFrame(test2, columns=[f'pca-{i}' for i in range(n_comp)])\n\n        train_features = pd.concat((train, train2), axis=1)\n        test_features = pd.concat((test, test2), axis=1)\n        return train_features, test_features\n    \n    def MinMax(self,train,test):\n        columns = [col for col in train.columns]\n        train_features = train[columns]\n        for col in tqdm(columns):\n            if len(Counter(train[col]))>2:\n                transformer = MinMaxScaler()#QuantileTransformer(n_quantiles=100,random_state=0, output_distribution=\"normal\")\n                vec_len = len(train_features[col].values)\n                vec_len_test = len(test[col].values)\n                raw_vec = train_features[col].values.reshape(vec_len, 1)\n                transformer.fit(raw_vec)\n                train_features[col] = transformer.transform(raw_vec).reshape(1, vec_len)[0]\n                test[col] = transformer.transform(test[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]\n        return train_features,test\n        \n    def read(self):\n        train = pd.read_csv(\"../input/smiles-bgu/tp53_features_train_smiles.csv\")\n        train['scaffold'] = train['ids'].apply(generate_scaffold)\n        test_features = pd.read_csv(\"../input/bgudrugchallenge2022/X_test.csv\")\n        labels= train['y']\n        columns = [col for col in train.columns if col.startswith('X')] + ['scaffold','ids']\n        train_features = train[columns]\n        groups = train_features['scaffold']\n        columns = [col for col in train.columns if col.startswith('X')]\n        train_features = train_features[columns]\n        test_features = test_features[columns]\n        print(\"run pca\")\n        train_features,test_features = self.pca(train_features,test_features)\n        print(\"run cluster\")\n        train_features,test_features = self.fe_cluster(train_features,test_features)\n        print(\"run stats\")\n        train_features,test_features = self.fe_stats(train_features,test_features)   \n        train_features,test_features = self.MinMax(train_features,test_features)\n#         sel = VarianceThreshold(threshold=0.005)\n#         sel.fit(train_features) \n#         new_col = [x for x in train_features.columns if x in train_features.columns[sel.get_support()]]\n#         train_features = train_features[new_col]\n#         test_features = test_features[new_col]\n        return train_features,test_features,train,labels,columns,groups\n        \n    def __call__(self):\n        if self.do_read_fast:\n            return self.read()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:04:15.499096Z","iopub.execute_input":"2022-06-27T17:04:15.499443Z","iopub.status.idle":"2022-06-27T17:04:15.538706Z","shell.execute_reply.started":"2022-06-27T17:04:15.499409Z","shell.execute_reply":"2022-06-27T17:04:15.537729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features,test_features,train,labels,columns,groups = MakeData().read()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:04:15.979598Z","iopub.execute_input":"2022-06-27T17:04:15.980164Z","iopub.status.idle":"2022-06-27T17:07:03.264093Z","shell.execute_reply.started":"2022-06-27T17:04:15.980126Z","shell.execute_reply":"2022-06-27T17:07:03.263026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def correlation(dataset, threshold):\n#     col_corr = set()  # Set of all the names of correlated columns\n#     corr_matrix = pd.DataFrame(np.corrcoef(dataset.values, rowvar=False), columns=dataset.columns)\n#     for i in tqdm(range(len(corr_matrix.columns))):\n#         for j in range(i):\n#             if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value\n#                 colname = corr_matrix.columns[i]  # getting the name of column\n#                 col_corr.add(colname)\n#     return col_corr\n\n# corr_features = correlation(train_features, 0.9)\n# print('correlated features: ', len(set(corr_features)) )","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:07:03.266322Z","iopub.execute_input":"2022-06-27T17:07:03.266690Z","iopub.status.idle":"2022-06-27T17:07:03.270985Z","shell.execute_reply.started":"2022-06-27T17:07:03.266653Z","shell.execute_reply":"2022-06-27T17:07:03.269968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_features.drop(labels=corr_features, axis=1, inplace=True)\n# test_features.drop(labels=corr_features, axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:07:03.272380Z","iopub.execute_input":"2022-06-27T17:07:03.273003Z","iopub.status.idle":"2022-06-27T17:07:03.285120Z","shell.execute_reply.started":"2022-06-27T17:07:03.272968Z","shell.execute_reply":"2022-06-27T17:07:03.284065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class trainer:\n### contains functions for training all models. Using this class you can train LGBM, PYTORCH, TABNET, STD and KERAS with bgu datasets.\n### In addition, LGBM HayperPrams can be optimized using the function \"lgbm_BayesianOptimization\"","metadata":{}},{"cell_type":"code","source":"import shap\nfrom itertools import cycle\nclass trainer():\n    def __init__(self,train_features,test_features,train,labels,columns,groups,folds=5):\n        self.train_features,self.test_features,self.train,self.labels,self.columns,self.groups = train_features,test_features,train,labels,columns,groups\n        self.folds = 5\n        self.features_to_remove = []\n        self.index_train = []\n        self.index_val = []\n        self.Kfold_cv(self.folds)\n        self.lgbBO = None\n        self.categorical_columns = [col for col in self.train_features if  self.train_features[col].nunique()==2 and \"X\" in col]\n        \n    def Kfold_cv(self,folds):\n        self.skf  = GroupKFold(n_splits=folds)\n        for train_idxs, val_idxs in self.skf.split(self.train_features, self.labels, self.groups):\n            self.index_train.append(train_idxs)\n            self.index_val.append(val_idxs)\n            \n    def train_nn_pytorch(self):\n        DEVICE = ('cuda' if torch.cuda.is_available() else 'cpu')\n        EPOCHS = 25\n        BATCH_SIZE = 128\n        LEARNING_RATE = 1e-4\n        WEIGHT_DECAY = 1e-5\n        EARLY_STOPPING_STEPS = 10\n        EARLY_STOP = False\n\n        num_features=len(self.train_features.columns)\n        num_targets=3\n        hidden_size=75\n        Y = []\n        y_val_pred = []\n        y_val_pred_prob = []\n        y_val_target = []\n        feature_imps = []\n        for fold_n ,(train_index, val_index) in enumerate(zip(self.index_train,self.index_val)):\n            acc_score_best = 0\n            x_train = self.train_features.loc[train_index].values\n            y_train = self.labels.loc[train_index].values\n            \n            x_valid = self.train_features.loc[val_index].values\n            y_valid = self.labels.loc[val_index].values\n\n            train_dataset = BGUDataset(x_train, y_train)\n            valid_dataset = BGUDataset(x_valid, y_valid)\n            trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n            validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n            model = Model(\n            num_features=num_features,\n            num_targets=num_targets,\n            hidden_size=hidden_size,)\n            model.to(DEVICE)\n            optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n            scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3, \n                                                      max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))\n\n            loss_fn = torch.nn.CrossEntropyLoss(weight=torch.from_numpy(np.array([1.0,0.7,1.0]))).cuda().float()#nn.CrossEntropyLoss()\n            early_stopping_steps = EARLY_STOPPING_STEPS\n            early_step = 0\n\n            for epoch in range(EPOCHS):\n                train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)\n#                 print(f\"EPOCH: {epoch}, train_loss: {train_loss}\")\n                valid_loss, valid_preds , labels = valid_fn(model, loss_fn, validloader, DEVICE)\n\n#                 print(f\" EPOCH: {epoch}, valid_loss: {valid_loss}\")\n#                 print(sum(valid_preds.argmax(1)==labels)/len(labels))\n                acc_score = sum(valid_preds.argmax(1)==labels)/len(labels)\n                if acc_score_best < acc_score:\n                    acc_score_best = acc_score\n                    torch.save(model.state_dict(), f\"FOLD.pth\")\n                    print(f\"fold_n: {fold_n}, epoch {epoch}, acc score improve -> \", sum(valid_preds.argmax(1)==labels)/len(labels))\n                    \n            model.load_state_dict(torch.load(f\"FOLD.pth\")) \n            valid_loss, valid_preds , labels = valid_fn(model, loss_fn, validloader, DEVICE)\n            y_val_pred_prob.append(valid_preds)\n            y_val_target.append(labels)\n            y_val_pred.append(valid_preds.argmax(1))\n            x_test = self.test_features.values\n            testdataset = TestDataset(x_test)\n            testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)\n            predictions = inference_fn(model, testloader, DEVICE)\n            Y.append(predictions)\n        y_val_pred = np.array(y_val_pred).reshape(-1)\n        y_val_target = np.array(y_val_target).reshape(-1)\n        print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return model,Y,[None]*5,y_val_pred_prob,y_val_target\n    \n    def train_cnn_pytorch(self):\n        DEVICE = ('cuda' if torch.cuda.is_available() else 'cpu')\n        EPOCHS = 25\n        BATCH_SIZE = 128\n        LEARNING_RATE = 1e-4\n        WEIGHT_DECAY = 1e-5\n        EARLY_STOPPING_STEPS = 10\n        EARLY_STOP = False\n\n        num_features=len(self.train_features.columns)\n        num_targets=3\n        hidden_size=75\n        hidden_size=64\n        Y = []\n        y_val_pred = []\n        y_val_pred_prob = []\n        y_val_target = []\n        feature_imps = []\n        for fold_n ,(train_index, val_index) in enumerate(zip(self.index_train,self.index_val)):\n            acc_score_best = 0\n            x_train = self.train_features.loc[train_index].values\n            y_train = self.labels.loc[train_index].values\n            \n            x_valid = self.train_features.loc[val_index].values\n            y_valid = self.labels.loc[val_index].values\n\n            train_dataset = BGUDataset(x_train, y_train)\n            valid_dataset = BGUDataset(x_valid, y_valid)\n            trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n            validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)\n    \n            model = ModelCNN(\n            num_features=num_features,\n            num_targets=num_targets,\n            hidden_size=hidden_size,)\n            model.to(DEVICE)\n            optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)\n            scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3, \n                                                      max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))\n\n#             loss_fn = torch.nn.CrossEntropyLoss(weight=torch.from_numpy(np.array([1.1,0.6,1.0]))).cuda().float()#nn.CrossEntropyLoss()\n            loss_fn = SmoothCrossEntropyLoss(weight=torch.from_numpy(np.array([0.9,0.6,1.2]))).cuda().float()\n            early_stopping_steps = EARLY_STOPPING_STEPS\n            early_step = 0\n\n\n            for epoch in range(EPOCHS):\n                train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)\n#                 print(f\"EPOCH: {epoch}, train_loss: {train_loss}\")\n                valid_loss, valid_preds , labels = valid_fn(model, loss_fn, validloader, DEVICE)\n\n#                 print(f\" EPOCH: {epoch}, valid_loss: {valid_loss}\")\n#                 print(sum(valid_preds.argmax(1)==labels)/len(labels))\n                acc_score = sum(valid_preds.argmax(1)==labels)/len(labels)\n                if acc_score_best < acc_score:\n                    acc_score_best = acc_score\n                    torch.save(model.state_dict(), f\"FOLD.pth\")\n                    print(f\"fold_n: {fold_n}, epoch {epoch}, acc score improve -> \", sum(valid_preds.argmax(1)==labels)/len(labels))\n                    \n            model.load_state_dict(torch.load(f\"FOLD.pth\")) \n            valid_loss, valid_preds , labels = valid_fn(model, loss_fn, validloader, DEVICE)\n            y_val_pred_prob.append(valid_preds)\n            y_val_target.append(labels)\n            y_val_pred.append(valid_preds.argmax(1))\n            x_test = self.test_features.values\n            testdataset = TestDataset(x_test)\n            testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)\n            predictions = inference_fn(model, testloader, DEVICE)\n            Y.append(predictions)\n        y_val_pred = np.array(y_val_pred).reshape(-1)\n        y_val_target = np.array(y_val_target).reshape(-1)\n        print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return model,Y,[None]*5,y_val_pred_prob,y_val_target\n    def train_TabNet(self,params=None,verbose = True):\n        \n        features = [ col for col in self.train_features.columns] \n        cat_idxs = [ i for i, f in enumerate(features) if f in self.categorical_columns]\n        cat_dims = [ 2 for i, f in enumerate(features) if f in self.categorical_columns]\n        \n        Y = []\n        y_val_pred = []\n        y_val_pred_prob = []\n        y_val_target = []\n        feature_imps = []\n        for train_index, val_index in zip(self.index_train,self.index_val):\n            \n            clf = TabNetClassifier(\n                    n_d=64, n_a=64, n_steps=5,\n                    gamma=1.5, n_independent=2, n_shared=2,\n                    cat_idxs=cat_idxs,\n                    cat_dims=cat_dims,\n                    cat_emb_dim=1,\n                    lambda_sparse=1e-4, momentum=0.3, clip_value=2.,\n                    optimizer_fn=torch.optim.AdamW,\n                    scheduler_params = {\"T_max\": 30},\n                    scheduler_fn=torch.optim.lr_scheduler.CosineAnnealingLR, epsilon=1e-15\n                )\n            \n            \n            clf.fit(\n                X_train = self.train_features.loc[train_index].values, \n                y_train = self.labels.loc[train_index].values,\n                eval_set=[(self.train_features.loc[train_index].values,self.labels.loc[train_index].values), \n                          (self.train_features.loc[val_index].values,self.labels.loc[val_index].values)],\n                eval_name=['train', 'valid'],\n                max_epochs=350, patience=100) \n            \n            y_val = clf.predict(self.train_features.values[val_index])\n            y_val_pred.append(y_val)\n            y_val_target.append(self.labels.values[val_index])\n            if verbose:\n                print(sum(y_val==self.labels.values[val_index])/len(y_val))\n            y_pred=clf.predict_proba(self.test_features.values)\n            Y.append(y_pred)\n            feature_imp = pd.DataFrame(sorted(zip(clf.feature_importances_,self.train_features.columns)), columns=['Value','Feature'])\n            feature_imps.append(feature_imp)\n        if verbose:\n            y_val_pred = np.array(y_val_pred).reshape(-1)\n            y_val_target = np.array(y_val_target).reshape(-1)\n            print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return clf,Y,feature_imps,y_val_pred,y_val_target\n    def train_lgbm(self,columns=None,params=None,verbose = True,do_shap=True):\n        if columns is None:\n            columns = [col for col in self.train_features.columns if \"X\" in col]\n        Y = []\n        y_val_pred = []\n        y_val_pred_proba = []\n        y_val_target = []\n        feature_imps = []\n        Shap_values = []\n        for train_index, val_index in zip(self.index_train,self.index_val):\n            if params is not None:\n                clf = lgb.LGBMClassifier(**params)\n            else:\n                clf = lgb.LGBMClassifier(n_estimators=1000,learning_rate=5e-2,max_depth=15,lambda_l2=0.5,class_weight={0:1.2,1:0.5,2:0.8})\n                \n#             params = {\n#                 'learning_rate': 0.01,\n#                 'metric': 'multiclass',\n#                 'objective': 'multiclass',\n#                 'num_classes': 3,\n#                 'max_depth': 15,\n#                 'learning_rate': 5e-2,\n#                 'lambda_l2': 0.5,\n#             }\n                \n#             train_set = lgb.Dataset(self.train_features[columns].loc[train_index], self.labels.loc[train_index] , categorical_feature = self.categorical_columns)\n#             val_set = lgb.Dataset(self.train_features[columns].loc[val_index],self.labels.loc[val_index], categorical_feature = self.categorical_columns)\n#             clf = lgb.train(params, train_set, num_boost_round = 100000, early_stopping_rounds = 100, \n#                           valid_sets=[train_set, val_set], verbose_eval = 100,class_weight = {0:1.2,1:0.5,2:0.8})\n            clf.fit(self.train_features[columns].loc[train_index],self.labels.loc[train_index],eval_set=[(self.train_features[columns].loc[val_index],self.labels.loc[val_index])],\n                    verbose=100,eval_metric='softmax',early_stopping_rounds=100,categorical_feature=[row for row in self.categorical_columns if row in columns])\n#             data_for_prediction = val_X.iloc[row_to_show]\n#             data_for_prediction_array = data_for_prediction.values.reshape(1, -1)\n            y_val = clf.predict(self.train_features[columns].values[val_index])\n            y_val_pred.append(y_val)\n            y_val_proba = clf.predict_proba(self.train_features[columns].values[val_index])\n            y_val_pred_proba.append(y_val_proba)\n            y_val_target.append(self.labels.values[val_index])\n            if verbose:\n                print(sum(y_val==self.labels.values[val_index])/len(y_val))\n            y_pred=clf.predict_proba(self.test_features[columns].values)\n            Y.append(y_pred)\n            if verbose:\n                lgb.plot_importance(clf,max_num_features=20)\n                plt.show()\n            feature_imp = pd.DataFrame(sorted(zip(clf.feature_importances_,self.train_features[columns].columns)), columns=['Value','Feature'])\n            feature_imps.append(feature_imp)\n            if do_shap:\n                explainer = shap.TreeExplainer(clf)\n                shap_values = explainer.shap_values(self.train_features[columns].loc[val_index].values)\n                shap.summary_plot(shap_values, self.train_features[columns].loc[val_index])\n                Shap_values.append(shap_values)\n                plt.show()\n#             shap.force_plot(explainer.expected_value[1], shap_values[1][0,:], self.train_features[columns].iloc[val_index[0],:])\n#             plt.show()\n        if verbose:\n            y_val_pred = np.array(y_val_pred).reshape(-1)\n            y_val_target = np.array(y_val_target).reshape(-1)\n            print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return clf,Y,feature_imps,y_val_pred,y_val_target,y_val_pred_proba,Shap_values\n    def train_nn(self,params=None):\n        \n        columns = [col for col in self.train_features.columns if \"pca\" not in col]#col.startswith('X')]\n        Y = []\n        y_val_pred = []\n        y_val_target = []\n        feature_imps = []\n        for train_index, val_index in zip(self.index_train,self.index_val):\n            encoded_els = Input((self.train_features[columns].shape[1],))\n            checkpoint_filepath = '/tmp/checkpoint'\n            model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n                filepath=checkpoint_filepath,\n                save_weights_only=True,\n                monitor='val_categorical_accuracy',\n                mode='max',\n                save_best_only=True,\n                verbose=2)\n            def scheduler(epoch, lr):\n                if epoch < 10:\n                    return lr\n                else:\n                    return lr * tf.math.exp(-0.2)\n            lrs = tf.keras.callbacks.LearningRateScheduler(scheduler)\n            layer1 = Dropout(0.5)(Dense(75, activation=\"relu\")(encoded_els))\n            layer1 = tf.keras.layers.BatchNormalization()(layer1)\n            out = Dense(3,activation=\"softmax\")(layer1)\n            clf = Model(inputs=encoded_els, outputs=[out])\n            clf.compile(optimizer=\"SGD\", loss_weights =np.array([1.2,0.8,1.1]) ,\n                        loss=weighted_categorical_crossentropy(np.array([1.1,0.8,1.0])),\n                        metrics=[tf.keras.metrics.CategoricalAccuracy()])\n            clf.fit(\n                self.train_features[columns].loc[train_index].values,\n                tf.keras.utils.to_categorical(self.labels.loc[train_index].values,num_classes=3),\n                epochs=50,\n                batch_size=32,\n                shuffle=True,\n                callbacks=[model_checkpoint_callback,lrs],\n                verbose=0,\n                validation_data=(self.train_features[columns].loc[val_index].values,tf.keras.utils.to_categorical(self.labels.loc[val_index].values,num_classes=3))\n            )\n            clf.load_weights(checkpoint_filepath)\n            y_pred=clf.predict(self.test_features[columns].values)\n            Y.append(y_pred)\n            \n            y_val = clf.predict(self.train_features[columns].values[val_index])\n            y_val_pred.append(y_val.argmax(1))\n            y_val_target.append(self.labels.values[val_index])\n            \n        y_val_pred = np.array(y_val_pred).reshape(-1)\n        y_val_target = np.array(y_val_target).reshape(-1)\n        print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return clf,Y,[None]*self.folds,y_val_pred,y_val_target \n            \n    def train_std(self,params=None,device=\"cuda:0\",verbose=True):\n        def acc(pred,label):\n            pred = torch.argmax(pred, dim=1)\n            return (pred==label).sum()/len(label)\n        \n        Y = []\n        y_val_pred = []\n        y_val_pred_p = []\n        y_val_target = []\n        feature_imps = []\n        for train_index, val_index in zip(self.index_train,self.index_val):\n            clf = STG(task_type='classification',input_dim=self.train_features.loc[train_index].shape[1], \n                      output_dim=3, hidden_dims=[100, 60], activation='tanh',\n                      optimizer='AdamW', learning_rate=1e-3, batch_size=512, \n                      feature_selection=True, sigma=1.5, lam=0.5, random_state=42, device=device) \n            clf.metric = acc\n            clf._model.loss = torch.nn.CrossEntropyLoss(weight=torch.from_numpy(np.array([1.6,0.8,1.2]))).cuda().float()\n            clf.fit(self.train_features.loc[train_index].values,self.labels.loc[train_index].values, \n                    nr_epochs=500, valid_X=self.train_features.loc[val_index].values, valid_y=self.labels.loc[val_index].values, \n                    print_interval=100,shuffle=True)\n            clf._model.eval()\n            clf._model.cpu()\n            y_val = clf.predict(self.train_features.values[val_index])\n            \n            y_val_pred.append(y_val)\n            y_val_target.append(self.labels.values[val_index])\n            if verbose:\n                print(sum(y_val==self.labels.values[val_index])/len(y_val))\n            X = dict(input=torch.from_numpy(self.test_features.values).float())\n            with torch.no_grad():\n                y_pred = clf._model(X)['prob'].detach().cpu().numpy()\n            Y.append(y_pred)\n        if verbose:\n            y_val_pred = np.array(y_val_pred).reshape(-1)\n            y_val_target = np.array(y_val_target).reshape(-1)\n            print(\"cv score : \", sum(y_val_pred==y_val_target)/len(y_val_target))\n        return clf,Y,[None]*self.folds,y_val_pred,y_val_target\n    \n    def SHAP(self,my_model,val_X):\n        explainer = shap.TreeExplainer(my_model)\n        shap_values = explainer.shap_values(val_X)\n        shap.summary_plot(shap_values[1], val_X)\n        plt.show()\n        shap.dependence_plot('Ball Possession %', shap_values[1], val_X, interaction_index=\"Goal Scored\")\n        plt.show()\n            \n    def train_xgboost(self,params):\n        pass\n    def train_catboost(self,params):\n        pass\n    \n    def plot_auc(self,y_test,y_score,title,n_classes=3):\n        lw = 2\n        fpr = dict()\n        tpr = dict()\n        roc_auc = dict()\n        for i in range(n_classes):\n            fpr[i], tpr[i], _ = roc_curve(y_test[:, i], y_score[:, i])\n            roc_auc[i] = auc(fpr[i], tpr[i])\n\n        fpr[\"micro\"], tpr[\"micro\"], _ = roc_curve(y_test.ravel(), y_score.ravel())\n        roc_auc[\"micro\"] = auc(fpr[\"micro\"], tpr[\"micro\"])\n        all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n        mean_tpr = np.zeros_like(all_fpr)\n        for i in range(n_classes):\n            mean_tpr += np.interp(all_fpr, fpr[i], tpr[i])\n\n        # Finally average it and compute AUC\n        mean_tpr /= n_classes\n\n        fpr[\"macro\"] = all_fpr\n        tpr[\"macro\"] = mean_tpr\n        roc_auc[\"macro\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n\n        # Plot all ROC curves\n        plt.figure()\n        plt.plot(\n            fpr[\"micro\"],\n            tpr[\"micro\"],\n            label=\"micro-average ROC curve (area = {0:0.2f})\".format(roc_auc[\"micro\"]),\n            color=\"deeppink\",\n            linestyle=\":\",\n            linewidth=4,\n        )\n\n        plt.plot(\n            fpr[\"macro\"],\n            tpr[\"macro\"],\n            label=\"macro-average ROC curve (area = {0:0.2f})\".format(roc_auc[\"macro\"]),\n            color=\"navy\",\n            linestyle=\":\",\n            linewidth=4,\n        )\n\n        colors = cycle([\"aqua\", \"darkorange\", \"cornflowerblue\"])\n        for i, color in zip(range(n_classes), colors):\n            plt.plot(\n                fpr[i],\n                tpr[i],\n                color=color,\n                lw=lw,\n                label=\"ROC curve of class {0} (area = {1:0.2f})\".format(i, roc_auc[i]),\n            )\n\n        plt.plot([0, 1], [0, 1], \"k--\", lw=lw)\n        plt.xlim([0.0, 1.0])\n        plt.ylim([0.0, 1.05])\n        plt.xlabel(\"False Positive Rate\")\n        plt.ylabel(\"True Positive Rate\")\n        plt.title(title)\n        plt.legend(loc=\"lower right\")\n        plt.show()\n        \n    def lgbm_BayesianOptimization(self):\n        \n        def lgb_eval(num_leaves, feature_fraction, bagging_fraction, max_depth, lambda_l1, lambda_l2, min_split_gain, min_child_weight,class_weight0,class_weight1,class_weight2):\n            params = {'n_estimators':1000, 'learning_rate':5e-2,'verbosity':-1}\n            params[\"num_leaves\"] = round(num_leaves)\n            params['feature_fraction'] = max(min(feature_fraction, 1), 0)\n            params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)\n            params['max_depth'] = round(max_depth)\n            params['lambda_l1'] = max(lambda_l1, 0)\n            params['lambda_l2'] = max(lambda_l2, 0)\n            params['min_split_gain'] = min_split_gain\n            params['min_child_weight'] = min_child_weight\n            params['class_weight'] = {0:class_weight0,1:class_weight1,2:class_weight2}\n            skf = GroupKFold(n_splits=5)\n            acc = []\n            for train_index, val_index in skf.split(train_features, labels,groups):\n                clf = lgb.LGBMClassifier(**params)\n                clf.fit(train_features.loc[train_index],labels.loc[train_index],eval_set=[(train_features.loc[val_index],labels.loc[val_index])],verbose=-1,eval_metric='softmax',early_stopping_rounds=100)\n                y_val = clf.predict(train_features.values[val_index])\n                acc.append(sum(y_val==labels.values[val_index])/len(y_val))\n            return np.mean(acc)\n        \n        self.lgbBO = BayesianOptimization(lgb_eval, {'num_leaves': (10, 45),\n                                        'feature_fraction': (0.1, 0.9),\n                                        'bagging_fraction': (0.8, 1),\n                                        'max_depth': (5, 20),\n                                        'lambda_l1': (0, 5),\n                                        'lambda_l2': (0, 3),\n                                        'min_split_gain': (0.001, 0.1),\n                                        'min_child_weight': (5, 50),\n                                        'class_weight0' : (1.1,1.3),\n                                        'class_weight1' : (0.6,0.7),\n                                        'class_weight2' : (0.9,1.1),\n                                       }, random_state=0)\n        self.lgbBO.maximize(init_points=15, n_iter=50)\n    def feature_selcet(self,feature_imps,th=5):\n        feature_imps_list = [feature_imps[row][feature_imps[row]['Value'] > th].Feature.values for row in range(5)]\n        feature_imps_list = np.concatenate(feature_imps_list).reshape(-1)\n        feature_imps_list = list(set(feature_imps_list))\n        return feature_imps_list\n        \n        \n\n        ","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:07:03.297825Z","iopub.execute_input":"2022-06-27T17:07:03.298770Z","iopub.status.idle":"2022-06-27T17:07:04.553919Z","shell.execute_reply.started":"2022-06-27T17:07:03.298728Z","shell.execute_reply":"2022-06-27T17:07:04.552981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run = trainer(train_features,test_features,train,labels,columns,groups)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:07:04.555586Z","iopub.execute_input":"2022-06-27T17:07:04.555950Z","iopub.status.idle":"2022-06-27T17:07:05.166372Z","shell.execute_reply.started":"2022-06-27T17:07:04.555915Z","shell.execute_reply":"2022-06-27T17:07:05.165378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# GBM - Train 5 folds\n* Training on raw data\n*  Select features using SHAP\n*  Re-training","metadata":{}},{"cell_type":"markdown","source":"## Step 1 - train on raw data","metadata":{}},{"cell_type":"code","source":"clf,Y_lgbm,feature_imps,y_val_pred,y_val_target,y_val_pred_proba,Shap_values = run.train_lgbm()\nrun.plot_auc(tf.keras.utils.to_categorical(y_val_target,num_classes=3),np.concatenate(y_val_pred_proba),n_classes=3,title='lgbm')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:07:05.167962Z","iopub.execute_input":"2022-06-27T17:07:05.168334Z","iopub.status.idle":"2022-06-27T17:12:11.034676Z","shell.execute_reply.started":"2022-06-27T17:07:05.168297Z","shell.execute_reply":"2022-06-27T17:12:11.033610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 2 - feature selcet","metadata":{}},{"cell_type":"code","source":"feature_selcet = run.feature_selcet(feature_imps)\nc = [col for i,col in enumerate(train_features.columns) if \"X\" in col]\nvals= np.abs(Shap_values).mean(0).mean(0).mean(0)\nfeature_importance = pd.DataFrame(list(zip(train_features[c].columns,vals)),columns=['col_name','feature_importance_vals'])\nfeature_importance.sort_values(by=['feature_importance_vals'],ascending=False,inplace=True)\nfeature_importance.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Step 3 - re-train","metadata":{}},{"cell_type":"code","source":"columns_shap = feature_importance.head(500).col_name\nclf_shap,Y_lgbm_shap,feature_imps_shap,y_val_pred_shap,y_val_target_shap,y_val_pred_proba_shap,_ = run.train_lgbm(columns_shap,do_shap=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:12:11.272646Z","iopub.execute_input":"2022-06-27T17:12:11.273286Z","iopub.status.idle":"2022-06-27T17:13:58.816359Z","shell.execute_reply.started":"2022-06-27T17:12:11.273247Z","shell.execute_reply":"2022-06-27T17:13:58.815346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GMB CV AUC","metadata":{}},{"cell_type":"code","source":"run.plot_auc(tf.keras.utils.to_categorical(y_val_target_shap,num_classes=3),np.concatenate(y_val_pred_proba_shap),n_classes=3,title='lgbm after Shap Feature Selection')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:13:58.823928Z","iopub.execute_input":"2022-06-27T17:13:58.826283Z","iopub.status.idle":"2022-06-27T17:13:59.119747Z","shell.execute_reply.started":"2022-06-27T17:13:58.826242Z","shell.execute_reply":"2022-06-27T17:13:59.118758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clf_fs,Y_lgbm_fs,feature_imps_fs,y_val_pred_fs,y_val_target_fs,y_val_pred_proba_fs,_ = run.train_lgbm(feature_selcet,do_shap=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:40:34.198771Z","iopub.execute_input":"2022-06-27T17:40:34.199281Z","iopub.status.idle":"2022-06-27T17:40:34.206788Z","shell.execute_reply.started":"2022-06-27T17:40:34.199238Z","shell.execute_reply":"2022-06-27T17:40:34.205739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# run.plot_auc(tf.keras.utils.to_categorical(y_val_target_fs,num_classes=3),np.concatenate(y_val_pred_proba_fs),n_classes=3,title='lgbm after Feature Selection')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:40:37.948767Z","iopub.execute_input":"2022-06-27T17:40:37.949288Z","iopub.status.idle":"2022-06-27T17:40:37.954819Z","shell.execute_reply.started":"2022-06-27T17:40:37.949240Z","shell.execute_reply":"2022-06-27T17:40:37.953712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch simple NN - Train 5 folds","metadata":{}},{"cell_type":"code","source":"model_nn,Y_nn,_,y_val_pred_prob_nn,y_val_target_nn = run.train_nn_pytorch()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:16:02.447508Z","iopub.execute_input":"2022-06-27T17:16:02.449785Z","iopub.status.idle":"2022-06-27T17:18:01.606125Z","shell.execute_reply.started":"2022-06-27T17:16:02.449746Z","shell.execute_reply":"2022-06-27T17:18:01.604713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show Pytorch simple NN block model","metadata":{}},{"cell_type":"code","source":"yhat = model_nn(torch.zeros([1,len(train_features.columns)]).float().cuda())\nmake_dot(yhat, params=dict(list(model_nn.named_parameters())))","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:27:28.582280Z","iopub.execute_input":"2022-06-27T17:27:28.582703Z","iopub.status.idle":"2022-06-27T17:27:28.687470Z","shell.execute_reply.started":"2022-06-27T17:27:28.582669Z","shell.execute_reply":"2022-06-27T17:27:28.686228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pytorch simple NN CV - AUC","metadata":{}},{"cell_type":"code","source":"run.plot_auc(tf.keras.utils.to_categorical(y_val_target_nn,num_classes=3),np.concatenate(y_val_pred_prob_nn),n_classes=3,title='pytorch simple nn')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:18:01.608372Z","iopub.execute_input":"2022-06-27T17:18:01.609450Z","iopub.status.idle":"2022-06-27T17:18:01.837045Z","shell.execute_reply.started":"2022-06-27T17:18:01.609404Z","shell.execute_reply":"2022-06-27T17:18:01.836140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pytorch CNN - Train 5 folds","metadata":{}},{"cell_type":"code","source":"model_cnn,Y_cnn,_,y_val_pred_prob_cnn,y_val_target_cnn = run.train_cnn_pytorch()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:18:01.838443Z","iopub.execute_input":"2022-06-27T17:18:01.839201Z","iopub.status.idle":"2022-06-27T17:20:57.507996Z","shell.execute_reply.started":"2022-06-27T17:18:01.839143Z","shell.execute_reply":"2022-06-27T17:20:57.506848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Show Pytorch CNN blocks model","metadata":{}},{"cell_type":"code","source":"yhat = model_cnn(torch.zeros([1,len(train_features.columns)]).float().cuda())\nmake_dot(yhat, params=dict(list(model_cnn.named_parameters())))","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:26:17.771801Z","iopub.execute_input":"2022-06-27T17:26:17.772335Z","iopub.status.idle":"2022-06-27T17:26:17.927288Z","shell.execute_reply.started":"2022-06-27T17:26:17.772281Z","shell.execute_reply":"2022-06-27T17:26:17.926089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pytorch CNN CV - AUC","metadata":{}},{"cell_type":"code","source":"run.plot_auc(tf.keras.utils.to_categorical(y_val_target_cnn,num_classes=3),np.concatenate(y_val_pred_prob_cnn),n_classes=3,title='pytorch cnn 1D')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:20:57.509679Z","iopub.execute_input":"2022-06-27T17:20:57.510294Z","iopub.status.idle":"2022-06-27T17:20:57.742034Z","shell.execute_reply.started":"2022-06-27T17:20:57.510252Z","shell.execute_reply":"2022-06-27T17:20:57.741096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble - Bayesian optimization\n#### Calculation the weights of the models according to optimization to maximize the accuracy of the CV score \n","metadata":{}},{"cell_type":"code","source":"from skopt import gp_minimize, forest_minimize\nfrom skopt.utils import use_named_args\nfrom skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret\nfrom skopt.space import Categorical, Integer, Real\n\ndef calculate_final_score_class(t,p0,p1,p2,p3,w0,w1,w2,w3):\n    acc = w0*p0 + w1*p1 + w2*p2 + w3*p3\n    return (acc.argmax(1)==t).sum()/(len(t))\ndef optimize(space, t,p0,p1,p2,p3, n_calls=50):\n    @use_named_args(space)\n    def score(**params):\n        final_score = calculate_final_score_class(t,p0,p1,p2,p3, **params)\n        return -final_score\n    return gp_minimize(func=score, dimensions=space, n_calls=n_calls)\nspace = [Real(0.1, 1.0, name='w0'),Real(0.1, 1.0, name='w1'),Real(0.95, 1.0, name='w2'),Real(0.1, 1.0, name='w3')]\nopt_result = optimize(space,y_val_target,np.concatenate(y_val_pred_prob_nn),np.concatenate(y_val_pred_prob_cnn),np.concatenate(y_val_pred_proba),np.concatenate(y_val_pred_proba_shap))","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:33:48.767385Z","iopub.execute_input":"2022-06-27T17:33:48.768156Z","iopub.status.idle":"2022-06-27T17:34:19.821572Z","shell.execute_reply.started":"2022-06-27T17:33:48.768115Z","shell.execute_reply":"2022-06-27T17:34:19.820504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ensemble - AUC","metadata":{}},{"cell_type":"code","source":"w0,w1,w2,w3 = np.array(opt_result.x)/sum(opt_result.x)\nprint(opt_result.x)\nprint(w0,w1,w2,w3)\nscore = -opt_result.fun\nprint(f\"Best accuracy CV - {score}\")\nval_ensemble = w0*np.concatenate(y_val_pred_prob_nn)+w1*np.concatenate(y_val_pred_prob_cnn)+w2*np.concatenate(y_val_pred_proba)+w3*np.concatenate(y_val_pred_proba_shap)\nrun.plot_auc(tf.keras.utils.to_categorical(y_val_target,num_classes=3),val_ensemble,n_classes=3,title='ensemble after optimize')","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:35:55.899084Z","iopub.execute_input":"2022-06-27T17:35:55.899760Z","iopub.status.idle":"2022-06-27T17:35:56.142297Z","shell.execute_reply.started":"2022-06-27T17:35:55.899718Z","shell.execute_reply":"2022-06-27T17:35:56.141195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Results\n| Model | CV Accuracy score| CV Auc Score | LB Accuracy Score |\n| --- | --- | --- |  --- |\n| LGBM - Raw Data | 0.562 | 0.74 | 0.552 |\n| LGBM - Raw Data - feature selection | 0.563 | 0.74 | 0.54X |\n| Pytorch CNN - Raw Data - PCA - K-means| 0.581 | 0.75 | 0.54X |\n| Pytorch NN - Raw Data - PCA - K-means| 0.582 | 0.74 | 0.54X |\n| ***Ensemple*** | ***0.591*** | ***0.76*** | ***0.5582*** |","metadata":{}},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/bgudrugchallenge2022/y_test_submission_example.csv\")\nsub['y_pred'] = (w2*np.array(Y_lgbm).mean(0) + w0 * np.array(Y_nn).mean(0) + w1* np.array(Y_cnn).mean(0)+ w3* np.array(Y_lgbm_shap).mean(0)).argmax(1)\nsub.to_csv(f\"sub{score}.csv\",index=False)\nCounter(sub['y_pred'])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:21:23.183447Z","iopub.execute_input":"2022-06-27T17:21:23.183810Z","iopub.status.idle":"2022-06-27T17:21:23.231411Z","shell.execute_reply.started":"2022-06-27T17:21:23.183772Z","shell.execute_reply":"2022-06-27T17:21:23.230357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = pd.read_csv(\"../input/bgudrugchallenge2022/y_test_submission_example.csv\")\nsub['y_pred'] = (0.97*np.array(Y_lgbm).mean(0) + 0.01 * np.array(Y_nn).mean(0) + 0.01* np.array(Y_cnn).mean(0)+ 0.01* np.array(Y_lgbm_shap).mean(0)).argmax(1)\nsub.to_csv(f\"sub2.csv\",index=False)\nCounter(sub['y_pred'])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T17:21:29.522605Z","iopub.execute_input":"2022-06-27T17:21:29.523448Z","iopub.status.idle":"2022-06-27T17:21:29.565732Z","shell.execute_reply.started":"2022-06-27T17:21:29.523406Z","shell.execute_reply":"2022-06-27T17:21:29.564701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Shap Explainer","metadata":{}},{"cell_type":"code","source":"shap.initjs()\nexplainer = shap.TreeExplainer(clf_shap)\nshap_values = explainer.shap_values(train_features[columns_shap])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:10:15.941282Z","iopub.execute_input":"2022-06-27T18:10:15.941730Z","iopub.status.idle":"2022-06-27T18:13:27.617376Z","shell.execute_reply.started":"2022-06-27T18:10:15.941691Z","shell.execute_reply":"2022-06-27T18:13:27.616274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## summary_plot","metadata":{}},{"cell_type":"code","source":"shap.summary_plot(shap_values, train_features[columns_shap])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize many predictions\n### visualize 1,000 individuals","metadata":{}},{"cell_type":"code","source":"shap.force_plot(explainer.expected_value[1], shap_values[1][:1000,:], train_features[columns_shap].iloc[:1000,:])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:13:44.330778Z","iopub.execute_input":"2022-06-27T18:13:44.331416Z","iopub.status.idle":"2022-06-27T18:13:49.416923Z","shell.execute_reply.started":"2022-06-27T18:13:44.331366Z","shell.execute_reply":"2022-06-27T18:13:49.412266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## visualize 3 individuals","metadata":{}},{"cell_type":"code","source":"shap.force_plot(explainer.expected_value[1], shap_values[1][:1,:], train_features[columns_shap].iloc[:1,:])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:15:14.959449Z","iopub.execute_input":"2022-06-27T18:15:14.960028Z","iopub.status.idle":"2022-06-27T18:15:14.998264Z","shell.execute_reply.started":"2022-06-27T18:15:14.959990Z","shell.execute_reply":"2022-06-27T18:15:14.997248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.force_plot(explainer.expected_value[1], shap_values[1][1:2,:], train_features[columns_shap].iloc[1:2,:])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:15:26.814897Z","iopub.execute_input":"2022-06-27T18:15:26.815458Z","iopub.status.idle":"2022-06-27T18:15:26.894307Z","shell.execute_reply.started":"2022-06-27T18:15:26.815413Z","shell.execute_reply":"2022-06-27T18:15:26.893392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"shap.force_plot(explainer.expected_value[1], shap_values[1][2:3,:], train_features[columns_shap].iloc[2:3,:])","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:15:40.261234Z","iopub.execute_input":"2022-06-27T18:15:40.261881Z","iopub.status.idle":"2022-06-27T18:15:40.300409Z","shell.execute_reply.started":"2022-06-27T18:15:40.261843Z","shell.execute_reply":"2022-06-27T18:15:40.299143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SHAP dependence plots show the effect of a single feature across the whole dataset\n## Show the 10 best feature","metadata":{}},{"cell_type":"code","source":"for name in columns_shap[0:10]:\n    shap.dependence_plot(name, shap_values[1],  train_features[columns_shap], display_features= train_features[columns_shap])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-06-27T18:16:29.371575Z","iopub.execute_input":"2022-06-27T18:16:29.371929Z","iopub.status.idle":"2022-06-27T18:18:43.594239Z","shell.execute_reply.started":"2022-06-27T18:16:29.371900Z","shell.execute_reply":"2022-06-27T18:18:43.593129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusions\n* LGBM gave the best LB result after manual calibration of the parameters. Calibration through optimization caused overtraining.\n* The parameter that gave the boost significantly was the weighting of the classes due to imbalance of training data.\n* Neuron networks gave the highest CV score but this did not rub off on the LB score.\n* Using a layer of Gaussian noise improved network learning and delayed entry into overtraining.\n* Use of SmoothCrossEntropyLoss has improved performance due to data imbalance.\n* Using SHAP helped slightly improve the CV but did not improve and even hit LB.\n* An ensemble of all models through weight optimization improved both CV and LB.\n* The main problem / main challenge was that the distribution of the data in the training is different from the public test and therefore it is very difficult to calibrate the parameters of the models\n* Another problem is that we do not know what each feature represents so it is difficult to do high quality pre-processing.","metadata":{}},{"cell_type":"markdown","source":"# What did not work\n* train a XGBoost, Catboost, keras NN, Std (Pytorch base), TabNet (pytorch base) gave less good results in CV and also in LB and did not help the ensemble\n* Pre-processing did not help GBM so I used raw data\n* Selecting features in neural networks hurt the results so I used all the features","metadata":{}},{"cell_type":"code","source":"# clf,Y,_,y_val_pred,y_val_target = run.train_TabNet()","metadata":{"execution":{"iopub.status.busy":"2022-06-26T18:18:42.382536Z","iopub.status.idle":"2022-06-26T18:18:42.383322Z","shell.execute_reply.started":"2022-06-26T18:18:42.383056Z","shell.execute_reply":"2022-06-26T18:18:42.38308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clf,Y,_,y_val_pred,y_val_target = run.train_std()","metadata":{"execution":{"iopub.status.busy":"2022-06-26T18:18:42.384738Z","iopub.status.idle":"2022-06-26T18:18:42.385503Z","shell.execute_reply.started":"2022-06-26T18:18:42.385248Z","shell.execute_reply":"2022-06-26T18:18:42.385271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}