{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport feather\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas()\n# from tqdm import tqdm\n\n# pd.options.display.max_rows = 999\n# pd.options.display.max_columns = 999\nimport glob\ndef get_path(str, first=True, parent_dir='../input/**/'):\n    res_li = glob.glob(parent_dir+str)\n    return res_li[0] if first else res_li","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/dogs-vs-cats-redux-kernels-edition/'\nevals = pd.read_csv('../input/dvc-prepare-evalset/evals.csv')\nevals['path'] = evals['path'].apply(lambda x: x.replace('../input/', DATA_DIR))\nevals.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"052c6b0c7448a11076309913be8f2116d9e3ac3c"},"cell_type":"code","source":"H, W, C = 150, 150, 3\nbatch_size = 32\neval_batch_size = batch_size * 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67431efa32ae1ebe3be5e4e80b6e5d0e005bd0fe"},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras.preprocessing.image import ImageDataGenerator\n\ntrain_gen = ImageDataGenerator(\n    rotation_range=20,\n    #width_shift_range=0.2,\n    #height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    #channel_shift_range=0.2,\n    horizontal_flip=True,\n    #vertical_flip=True,\n    #rescale=1./255,#!!!!NO!\n    preprocessing_function=lambda x:(x-x.mean())/x.std()\n)\ntest_gen = ImageDataGenerator(\n    #rescale=1./255,\n    preprocessing_function=lambda x:(x-x.mean())/x.std()\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0e1381b9768e52ac2e925f2d6060cc57545f38d0"},"cell_type":"code","source":"def get_filenames_targets(eval_mode, valid_fold, n_valid, evals=evals):\n    if eval_mode=='train':\n        mask = (evals['is_test']==0) & (evals['eval_set']!=valid_fold)\n    elif eval_mode=='valid':\n        mask = (evals['is_test']==0) & (evals['eval_set']==valid_fold)\n    elif eval_mode=='test':\n        mask = (evals['is_test']==1)\n    else:\n        raise NotImplementedError\n    filenames_arr = evals.loc[mask, 'path'].values\n    target_arr = evals.loc[mask, 'target'].values\n    return filenames_arr, target_arr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4e15b5c483b73f1bb22ffe1931031e67c0a726c7"},"cell_type":"markdown","source":"## Customized ImageIterator\n- https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/image/Iterator\n- https://www.kaggle.com/aloisiodn/fast-thread-safe-keras-generator-from-bin-files\n- https://anandology.com/blog/using-iterators-and-generators/"},{"metadata":{"trusted":true,"_uuid":"c04f88866736fb10a2b833584d81666b726a20bf","collapsed":true},"cell_type":"code","source":"import keras.backend as K\nfrom keras.preprocessing.image import Iterator\nfrom keras.preprocessing.image import load_img, img_to_array\nimport threading\n\nclass ImageIterator(Iterator):\n    def __init__(\n        self, img_generator, \n        eval_mode, valid_fold, n_valid=128*16, \n        evals=evals, \n        target_size=(H, W),\n        num_class=1,\n        batch_size=batch_size,\n        shuffle=False,\n        use_tta='fliplr',\n        seed=42\n    ):\n        shuffle = True if eval_mode=='train' else False\n        filenames_arr, target_arr = get_filenames_targets(\n            eval_mode, valid_fold, n_valid, evals=evals\n        )\n        if eval_mode=='valid' and n_valid is not None:\n            filenames_arr = filenames_arr[:n_valid]\n            target_arr = target_arr[:n_valid]\n        if shuffle:\n            indexes = np.arange(flow.samples)\n            np.random.permutatione(indexes)\n            filenames_arr = filenames_arr[indexes]\n            target_arr = target_arr[indexes]\n        assert len(filenames_arr)==len(target_arr)\n        n = len(filenames_arr)\n        \n        self.img_generator = img_generator\n        self.class_indices = {'dog': 0, 'cat': 1}\n        self.eval_mode = eval_mode\n        self.n = n\n        self.target_size = target_size\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.filenames = filenames_arr\n        self.classes = target_arr\n        self.num_class = num_class\n        self.seed = seed\n        self.use_tta = use_tta\n        self.lock = threading.Lock()\n        \n        super(ImageIterator, self).__init__(\n            n=n, batch_size=batch_size, shuffle=shuffle, seed=seed\n        )\n    \n    def _get_batches_of_transformed_samples(self, index_array):\n        X = np.zeros((len(index_array),) + (H, W, C), dtype=K.floatx())\n        Y = np.zeros((len(index_array), self.num_class), dtype=K.floatx())\n        \n        for i, idx in enumerate(index_array):\n            with self.lock:\n                x = load_img(\n                    path=self.filenames[idx],\n                    target_size=self.target_size\n                )\n                x = img_to_array(x)\n                \n            if self.use_tta=='fliplr':\n                x = x[:, ::-1, :].copy()\n                \n            X[i] = x.astype(K.floatx())\n            Y[i] = self.classes[idx].astype(K.floatx())\n        ### latest keras version supports ImageDataGenerator.apply_transform\n        return next(self.img_generator.flow(\n           X, Y, batch_size=len(index_array), shuffle=False            \n        ))\n    \n    def next(self):\n        with self.lock: \n            index_array = next(self.index_generator)\n        return self._get_batches_of_transformed_samples(index_array)       ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43111048b28fbef095633edf41fe13a6660d98c1"},"cell_type":"markdown","source":"## Test Time Augmentation\n- pred_normal_image, pred_fliplr_image(, pred...) and take ensemble(average)"},{"metadata":{"trusted":true,"_uuid":"17dbc496b1446f963c8c862bc9880d9a839e8b1f","collapsed":true},"cell_type":"code","source":"train_flow = ImageIterator(train_gen, 'valid', 1, batch_size=batch_size, use_tta=None)\nvalid_flow = ImageIterator(test_gen, 'valid', 0, batch_size=eval_batch_size, use_tta=None)\nvalid_tta_flow = ImageIterator(test_gen, 'valid', 0, batch_size=eval_batch_size, use_tta='fliplr')\ntest_flow = ImageIterator(test_gen, 'test', None, batch_size=eval_batch_size, use_tta=None)\ntest_tta_flow = ImageIterator(test_gen, 'test', None, batch_size=eval_batch_size, use_tta='fliplr')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3798de263fe2cad4e8d11fe6fec161c5a52f8e2d"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.models import Model\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping\nfrom keras import optimizers, losses, activations, models\nfrom keras.layers import Conv2D, Dense, Input, Flatten, Concatenate, Dropout, Activation\nfrom keras.layers import BatchNormalization, MaxPooling2D, GlobalAveragePooling2D\nfrom keras import applications","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"af5cab2e4b56ef25bdb0c0a7e399ae8228e606e5"},"cell_type":"code","source":"n_final_state = 32\n\ndef get_model(n_final_state, lr=1e-3, decay=1e-8):\n    input_shape = (H, W, C)\n    \n    input_x = Input(shape=input_shape)\n    \n    c1 = Conv2D(32, (3, 3))(input_x)\n    c1 = BatchNormalization()(c1)\n    c1 = Activation('relu')(c1)\n    c1 = MaxPooling2D((2, 2))(c1)\n    \n    c2 = Conv2D(32, (3, 3))(c1)\n    c2 = BatchNormalization()(c2)\n    c2 = Activation('relu')(c2)\n    c2 = MaxPooling2D((2, 2))(c2)\n    \n    c3 = Conv2D(64, (3, 3))(c2)\n    c3 = BatchNormalization()(c3)\n    c3 = Activation('relu')(c3)\n    c3 = MaxPooling2D((2, 2))(c3)\n    \n    flat = Flatten()(c3)\n    \n    d1 = Dense(\n        64, activation='relu'\n    )(flat)\n    #d1 = Dropout(0.5)(d1)\n    d1 = BatchNormalization()(d1)\n    \n    final_state = Dense(\n        n_final_state, activation='relu', name='final_state'\n    )(d1)\n    \n    x = Dropout(0.5)(final_state)\n    outputs = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=input_x, outputs=outputs)\n    optimizer=optimizers.Adam(lr=lr, decay=decay)\n    model.compile(loss='binary_crossentropy', \n                  optimizer=optimizer,\n                  metrics=['accuracy'])\n    return model\n\nmodel = get_model(n_final_state=n_final_state)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19a1e9fae25cd5e012d1814db7f720f2375aa504"},"cell_type":"code","source":"train_steps = int(np.ceil(train_flow.n / batch_size))\nvalid_steps = int(np.ceil(valid_flow.n / eval_batch_size))\ntest_steps = int(np.ceil(test_flow.n / eval_batch_size))\nprint(f'train {train_steps} steps')\nprint(f'valid {valid_steps} steps')\nprint(f'test {test_steps} steps')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a18ee26f76cc31b3f30eab311530b46b7ec10b91"},"cell_type":"markdown","source":"## Snapshot Ensemble&Cyclic Learning Rate\n- https://github.com/titu1994/Snapshot-Ensembles"},{"metadata":{"trusted":true,"_uuid":"86674a0be56faa7652ae222d170827b8d6f32947","collapsed":true},"cell_type":"code","source":"## https://github.com/keras-team/keras-contrib/blob/master/keras_contrib/callbacks/snapshot.py\n## https://github.com/keras-team/keras/blob/master/keras/callbacks.py#L146\nfrom keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler\n\nclass SnapshotModelCheckpoint(Callback):\n    \"\"\"Callback that saves the snapshot weights of the model.\n    Saves the model weights on certain epochs (which can be considered the\n    snapshot of the model at that epoch).\n    Should be used with the cosine annealing learning rate schedule to save\n    the weight just before learning rate is sharply increased.\n    # Arguments:\n        nb_epochs: total number of epochs that the model will be trained for.\n        nb_snapshots: number of times the weights of the model will be saved.\n        fn_prefix: prefix for the filename of the weights.\n    \"\"\"\n\n    def __init__(self, nb_epochs, nb_snapshots, fn_prefix='Model', verbose=1):\n        super(SnapshotModelCheckpoint, self).__init__()\n\n        self.check = nb_epochs // nb_snapshots\n        self.fn_prefix = fn_prefix\n        self.verbose = verbose\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch != 0 and (epoch + 1) % self.check == 0:\n            filepath = self.fn_prefix + \"-%d.h5\" % ((epoch + 1) // self.check)\n            self.model.save_weights(filepath, overwrite=True)\n            if self.verbose>0:\n                print(\"Saved snapshot at weights/%s_%d.h5\" % (self.fn_prefix, epoch))\n                \nclass SnapshotCallbackBuilder:\n    \"\"\"Callback builder for snapshot ensemble training of a model.\n    From the paper \"Snapshot Ensembles: Train 1, Get M For Free\" (https://openreview.net/pdf?id=BJYwwY9ll)\n    Creates a list of callbacks, which are provided when training a model\n    so as to save the model weights at certain epochs, and then sharply\n    increase the learning rate.\n    \"\"\"\n\n    def __init__(self, nb_epochs, nb_snapshots, init_lr=0.1, verbose=1):\n        \"\"\"\n        Initialize a snapshot callback builder.\n        # Arguments:\n            nb_epochs: total number of epochs that the model will be trained for.\n            nb_snapshots: number of times the weights of the model will be saved.\n            init_lr: initial learning rate\n        \"\"\"\n        self.T = nb_epochs\n        self.M = nb_snapshots\n        self.alpha_zero = init_lr\n        self.verbose = verbose\n\n    def get_callbacks(self, model_prefix='Model'):\n        \"\"\"\n        Creates a list of callbacks that can be used during training to create a\n        snapshot ensemble of the model.\n        Args:\n            model_prefix: prefix for the filename of the weights.\n        Returns: list of 3 callbacks [ModelCheckpoint, LearningRateScheduler,\n                 SnapshotModelCheckpoint] which can be provided to the 'fit' function\n        \"\"\"\n        if not os.path.exists('weights/'):\n            os.makedirs('weights/')\n\n        callback_list = [\n            ModelCheckpoint('weights/%s-Best.h5' % model_prefix, monitor='val_acc',\n                            save_best_only=True, save_weights_only=True),\n            LearningRateScheduler(schedule=self._cosine_anneal_schedule),\n            SnapshotModelCheckpoint(\n                self.T, self.M, fn_prefix='weights/%s' % model_prefix, verbose=self.verbose\n            )\n        ]\n\n        return callback_list\n\n    def _cosine_anneal_schedule(self, t):\n        cos_inner = np.pi * (t % (self.T // self.M))\n        cos_inner /= self.T // self.M\n        cos_out = np.cos(cos_inner) + 1\n        return float(self.alpha_zero / 2 * cos_out)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fbb0a70d7cff24f0a718562e18896597913899c","_kg_hide-output":true,"scrolled":false},"cell_type":"code","source":"epochs = 50\n\nprint('BATCH_SIZE: {} EPOCHS: {}'.format(batch_size, epochs))\n\nfile_path='model.h5'\ncheckpoint = ModelCheckpoint(\n    file_path, monitor='val_loss', verbose=1, \n    save_best_only=True, \n    save_weights_only=True,\n    mode='min'\n)\nearly = EarlyStopping(monitor='val_loss', mode='min', patience=30)\n#callbacks_list = [checkpoint, early]\n#K.set_value(model.optimizer.lr, 0.0005)\nsnapshot_cb = SnapshotCallbackBuilder(epochs, 10, 0.0005*2)\ncallbacks_list = snapshot_cb.get_callbacks(model_prefix='Model')\n\ngc.collect();\nhistory = model.fit_generator(\n    train_flow, \n    steps_per_epoch=train_steps,\n    validation_data=valid_flow,\n    validation_steps=valid_steps,\n    epochs=epochs, \n    verbose=1,\n    callbacks=callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bc882f97f023a58b10563881bd9f3a36530bb23","collapsed":true},"cell_type":"code","source":"eval_res = pd.DataFrame(history.history)\neval_res.to_csv('eval_res.csv', index=False)\nfor c in ['acc', 'loss']:\n    eval_res[[c, f'val_{c}']].plot(figsize=[18, 6]);\n    plt.xlabel('Epoch'); plt.ylabel(c);\n    plt.title(c); plt.grid();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f359b8d15c1c0226e4d203cd36331f830c59b20e"},"cell_type":"code","source":"def predict(model, modelpath, data_flow, steps, workers=4, verbose=1):\n    model.load_weights(modelpath)\n    pred = model.predict_generator(\n        generator=data_flow,\n        steps=steps, \n        use_multiprocessing=True  if workers>1 else False, \n        workers=workers, \n        verbose=verbose\n    )\n    return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62d5d731fd659e3e33e233987041bbe506db7deb","collapsed":true},"cell_type":"code","source":"glob.glob('./weights/Model*.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"274c458700594ca1d76f0ba8052cb922897236fc","collapsed":true},"cell_type":"code","source":"pred_val_best = predict(\n    model, './weights/Model-Best.h5', \n    valid_flow, valid_steps, workers=4\n)\npred_val_best_tta = predict(\n    model, './weights/Model-Best.h5', \n    valid_tta_flow, valid_steps, workers=4\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"412301f25bef793ee1e6e81599c7ab700c0daaaf","collapsed":true},"cell_type":"code","source":"print(pred_val_best.shape, pred_val_best_tta.shape)\nsns.distplot(pred_val_best)\nsns.distplot(pred_val_best_tta)\nplt.legend(['normal', 'fliplr']);\nplt.grid();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1be72f9bdbd44be10556739f7f4802c8012ca62b","collapsed":true},"cell_type":"code","source":"from sklearn.metrics import log_loss, accuracy_score\npred_val_best = pred_val_best.ravel()\npred_val_best_tta = pred_val_best_tta.ravel()\ny_valid =  valid_flow.classes.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f02cd74515656fe5c0e5fc93e076c71d0a27a1","collapsed":true},"cell_type":"code","source":"val_loss = log_loss(y_valid, pred_val_best)\nval_acc = accuracy_score(y_valid, np.round(pred_val_best))\nprint(f'normal \\nvalid loss: {val_loss:.6f}\\t valid accuracy: {val_acc:.4%}')\nval_loss = log_loss(y_valid, pred_val_best_tta)\nval_acc = accuracy_score(y_valid, np.round(pred_val_best_tta))\nprint(f'tta \\nvalid loss: {val_loss:.6f}\\t valid accuracy: {val_acc:.4%}')\n\npred_val_best_avg = (pred_val_best+pred_val_best_tta)/2.\n\nval_loss = log_loss(y_valid, pred_val_best_avg)\nval_acc = accuracy_score(y_valid, np.round(pred_val_best_avg))\nprint(f'avg(normal tta) \\nvalid loss: {val_loss:.6f}\\t valid accuracy: {val_acc:.4%}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddbd0eb350ce35fd0a87a6401c621c7561567ee0","collapsed":true},"cell_type":"code","source":"print(sorted(glob.glob('./weights/Model*'))[2:-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d6c1e58e10810c192f2e40d361aa8756636aee6","collapsed":true},"cell_type":"code","source":"pred_val_li = [predict(\n    model, p, valid_flow, valid_steps, workers=4, verbose=1\n) for p in sorted(glob.glob('./weights/Model*'))[2:-1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7aeea6e0a9c222bad2bec7e9d560b432f730fa09","collapsed":true},"cell_type":"code","source":"pred_val_tta_li = [predict(\n    model, p, valid_tta_flow, valid_steps, workers=4, verbose=1\n) for p in sorted(glob.glob('./weights/Model*'))[2:-1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58de64c82277c6da76fd6fe93fe915760c9acb26","collapsed":true},"cell_type":"code","source":"for i,(p, p_tta) in enumerate(zip(pred_val_li, pred_val_tta_li)):\n    print(i+2, 'th snapshot normal loss: {:.6f} acc: {:.6f}'.format(\n        log_loss(y_valid, p),\n        accuracy_score(y_valid, np.round(p))\n    ))\n    print(i+2, 'th snapshot tta    loss: {:.6f} acc: {:.6f}'.format(\n        log_loss(y_valid, p_tta),\n        accuracy_score(y_valid, np.round(p_tta))\n    ))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cc91dd18161661470b64a506dc199278cb21638","collapsed":true},"cell_type":"code","source":"X_meta = pred_val_li + pred_val_tta_li\nX_meta = np.hstack(X_meta)\nX_meta.shape","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true,"_uuid":"cb81ec7f5a585a8981a3b6733997664818eb24ad","collapsed":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegressionCV\nmeta_model = LogisticRegressionCV(scoring='neg_log_loss')\nmeta_model.fit(X_meta, y_valid)\nprint(meta_model.coef_, meta_model.intercept_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1242f81bbae8a493d360524a6e7301d3b9e4825f","collapsed":true},"cell_type":"code","source":"pred_val_ens_meta = meta_model.predict_proba(X_meta)[:, 1]\nprint('snapshot-ens meta loss: {:.6f} acc: {:.6f}'.format(\n    log_loss(y_valid, pred_val_ens_meta),\n    accuracy_score(y_valid, np.round(pred_val_ens_meta))\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97d7d2b00661bccf6b71538f7bcaaa2ae4737b18","collapsed":true},"cell_type":"code","source":"pred_val_ens_avg = X_meta[:, 3:].mean(1)\nprint('snapshot-ens avg loss: {:.6f} acc: {:.6f}'.format(\n    log_loss(y_valid, pred_val_ens_avg),\n    accuracy_score(y_valid, np.round(pred_val_ens_avg))\n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5da6cb365708f551fc603d25a9a84ac3da017c60","collapsed":true},"cell_type":"code","source":"pred_test_best = predict(\n    model, './weights/Model-Best.h5', \n    test_flow, test_steps, workers=4\n)\npred_test_best_tta = predict(\n    model, './weights/Model-Best.h5', \n    test_tta_flow, test_steps, workers=4\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60009d5d1761962d5a035ba38a221a06def1a383","collapsed":true},"cell_type":"code","source":"%%time\npred_test_li = [predict(\n    model, p, test_flow, test_steps, workers=4, verbose=1\n) for p in sorted(glob.glob('./weights/Model*'))[2:-1]]\npred_test_tta_li = [predict(\n    model, p, test_tta_flow, test_steps, workers=4, verbose=1\n) for p in sorted(glob.glob('./weights/Model*'))[2:-1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"36a08ccbcc7cd3bba56e4b8d718341cdc834e827"},"cell_type":"code","source":"pred_test_best = pred_test_best.ravel()\npred_test_best_tta = pred_test_best_tta.ravel()\nX_meta_test = pred_test_li + pred_test_tta_li\nX_meta_test = np.hstack(X_meta_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"123d968e8eb485062c64e1ebd0eac477afdbea6f"},"cell_type":"code","source":"pred_test_best_avg = (pred_test_best+pred_test_best_tta)/2.\npred_test_ens_meta = meta_model.predict_proba(X_meta_test)[:, 1]\npred_test_ens_avg = X_meta_test[:, 3:].mean(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad1dfbfd1b90687c265de84e602224e633878a5a","collapsed":true},"cell_type":"code","source":"def make_sub(pred_test, name, val_loss, evals=evals):\n    mask = evals['is_test']==1\n    sub = {\n        'id': evals.loc[mask, 'img_id'].values.astype('int'),\n        'label': pred_test,\n    }\n    sub = pd.DataFrame(sub).sort_values(by='id').reset_index(drop=True)\n    sub['label'] = 1 - sub['label']\n    subname = f'{name}_{val_loss:.6f}.csv'\n    sub.to_csv(subname, index=False)\n    print(subname, 'saved')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0accc6756777fce86cb3cdb70f5dd002d93bbb21","collapsed":true},"cell_type":"code","source":"for pair in [\n    ('best',     pred_test_best,     pred_val_best),\n    ('best_tta', pred_test_best_tta, pred_val_best_tta),\n    ('best_avg', pred_test_best_avg, pred_val_best_avg), \n    ('ens_meta', pred_test_ens_meta, pred_val_ens_meta),\n    ('ens_avg',  pred_test_ens_avg,  pred_val_ens_avg)\n]:\n    name, p_t, p_v = pair\n    val_loss = log_loss(y_valid, p_v)\n    make_sub(p_t, name, val_loss, evals=evals)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}