{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport random\nimport cv2\nfrom sklearn.model_selection import KFold, train_test_split\nimport tabulate\nimport pickle\nimport shutil\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport keras.backend as K\n\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.preprocessing.image import *","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-04T06:42:31.504322Z","iopub.execute_input":"2022-03-04T06:42:31.504587Z","iopub.status.idle":"2022-03-04T06:42:31.510844Z","shell.execute_reply.started":"2022-03-04T06:42:31.504557Z","shell.execute_reply":"2022-03-04T06:42:31.509949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNIST:\n    def __init__(self, random_state=7):\n        self.x_train, self.y_train = None, None\n        self.x_valid, self.y_valid = None, None\n        self.x_test, self.y_test = None, None\n\n        self.random_state = random_state\n        self.set_random_state(self.random_state)\n\n    def load_data_from_local(self, data_path=r'd:\\resources\\data\\mnist'):\n        train_df = pd.read_csv(os.path.join(data_path, \"train.csv\"))\n        test_df = pd.read_csv(os.path.join(data_path, \"test.csv\"))\n\n        x_train = train_df.iloc[:, 1:].values\n        y_train = train_df.iloc[:, 0].values\n        x_test = test_df.iloc[:, :].values\n        y_test = None\n\n        x_train = (x_train - 127.5) / 127.5\n        x_test = (x_test - 127.5) / 127.5\n\n        x_train = np.reshape(x_train, (-1, 28, 28, 1))\n        x_test = np.reshape(x_test, (-1, 28, 28, 1))\n\n        self.x_train = x_train\n        self.y_train = y_train\n        self.x_test = x_test\n        self.y_test = y_test\n\n        return self.x_train, self.y_train, self.x_test, self.y_test\n\n    def exclude(self, indexes):\n        self.x_train = np.delete(self.x_train, indexes, axis=0)\n        self.y_train = np.delete(self.y_train, indexes, axis=0)\n\n    def extend_to_3_channels(self):\n        x_train = np.stack([self.x_train, self.x_train, self.x_train], axis=3)\n        self.x_train = np.reshape(x_train, (-1, 28, 28, 3))\n\n        x_test = np.stack([self.x_test, self.x_test, self.x_test], axis=3)\n        self.x_test = np.reshape(x_test, (-1, 28, 28, 3))\n\n    def extend_img_with_rotations(self, x, rotation):\n        nx = np.zeros((28*2, 28*2))\n\n        x = np.reshape(x, (28, 28))\n        x = x.astype(np.uint8)\n        nx[:28, :28] = x\n\n        M = cv2.getRotationMatrix2D((14, 14), rotation, 1.0)\n        x1 = cv2.warpAffine(x, M, (28, 28))\n        nx[28:, :28] = x1\n\n        M = cv2.getRotationMatrix2D((14, 14), -rotation, 1.0)\n        x2 = cv2.warpAffine(x, M, (28, 28))\n        nx[:28, 28:] = x2\n\n        return nx\n\n    def set_random_state(self, random_state):\n        tf.random.set_seed(random_state)\n        np.random.seed(random_state)\n        random.seed(random_state)\n\n    def folds_split(self, n_splits=10, shuffle=True):\n        kfold = KFold(n_splits=n_splits, shuffle=shuffle, random_state=self.random_state)\n        fold_idx = list(kfold.split(self.x_train))\n        for i, (train_idx, valid_idx) in enumerate(fold_idx):\n            x_train_fold, y_train_fold = self.x_train[train_idx, :, :, :], self.y_train[train_idx]\n            x_valid_fold, y_valid_fold = self.x_train[valid_idx, :, :, :], self.y_train[valid_idx]\n\n            yield x_train_fold, y_train_fold, x_valid_fold, y_valid_fold, i\n\n    def random_split(self, test_size=0.2, shuffle=True):\n        return train_test_split(self.x_train, self.y_train,\n                                test_size=test_size, shuffle=shuffle, random_state=self.random_state)\n\n    def summary(self):\n        print(\"x_train shape = \", self.x_train.shape if self.x_train is not None else None)\n        print(\"y_train shape = \", self.y_train.shape if self.y_train is not None else None)\n        print(\"x_valid shape = \", self.x_valid.shape if self.x_valid is not None else None)\n        print(\"y_valid shape = \", self.y_valid.shape if self.y_valid is not None else None)\n        print(\"x_test  shape = \", self.x_test.shape if self.x_test is not None else None)\n        print(\"y_test  shape = \", self.y_test.shape if self.y_test is not None else None)\n","metadata":{"execution":{"iopub.status.busy":"2022-03-04T06:42:31.512479Z","iopub.execute_input":"2022-03-04T06:42:31.512714Z","iopub.status.idle":"2022-03-04T06:42:31.538330Z","shell.execute_reply.started":"2022-03-04T06:42:31.512665Z","shell.execute_reply":"2022-03-04T06:42:31.537578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TrainLossStopping(Callback):\n    def __init__(self, min_loss):\n        super(TrainLossStopping, self).__init__()\n        self.min_loss = min_loss\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        current = logs.get('loss')\n        if current is None:\n            return\n\n        if current <= self.min_loss:\n            self.model.stop_training = True\n\n\nclass ModelCheckpointV2(Callback):\n    def __init__(self, filepath):\n        super(ModelCheckpointV2, self).__init__()\n        self.filepath = filepath\n        self.min_val_loss = None\n        self.max_val_acc = None\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        val_loss = logs.get('val_loss')\n        val_acc = logs.get('val_acc')\n\n        if val_loss is None or val_acc is None:\n            return\n\n        if self.max_val_acc is None or self.max_val_acc < val_acc or (self.max_val_acc == val_acc and self.min_val_loss > val_loss):\n            self.max_val_acc = val_acc\n            self.min_val_loss = val_loss\n            print(\"saving model to {}, val_loss = {}, val_acc = {}\".format(self.filepath, val_loss, val_acc))\n            self.model.save(self.filepath)\n\n\nclass EarlyStoppingV2(Callback):\n    def __init__(self, patience=10):\n        super(EarlyStoppingV2, self).__init__()\n        self.patience = patience\n        self.min_loss = None\n        self.min_val_loss = None\n        self.loss_patience = 0\n        self.val_loss_patience = 0\n\n    def on_epoch_end(self, epoch, logs=None):\n        logs = logs or {}\n        loss = logs.get('loss')\n        val_loss = logs.get('val_loss')\n\n        if loss is None or val_loss is None:\n            return\n\n        if self.min_loss is None or loss < self.min_loss:\n            self.min_loss = loss\n            self.loss_patience = 0\n        elif loss >= self.min_loss:\n            self.loss_patience += 1\n\n        if self.min_val_loss is None or val_loss < self.min_val_loss:\n            self.min_val_loss = val_loss\n            self.val_loss_patience = 0\n        elif val_loss >= self.min_val_loss:\n            self.val_loss_patience += 1\n\n        if self.loss_patience >= 5 or self.val_loss_patience >= 5:\n            print(\"loss_patience = \", self.loss_patience, \"val_loss_patience = \", self.val_loss_patience)\n\n        if self.loss_patience >= self.patience and self.val_loss_patience >= self.patience:\n            self.model.stop_training = True\n\n\nclass Net:\n\n    def __init__(self, name, description=None, random_state=7, model_base=\"f:/temp/models\"):\n        self.name = name\n        self.description = description if description is not None else self.name\n\n        self.random_state = random_state\n        self.set_random_state(random_state)\n\n        self.models = []\n        self.histories = []\n\n        self.model_path = os.path.join(model_base, self.name)\n        self.create_path(os.path.join(self.model_path, \".gitempty\"))\n\n    def reset(self):\n        self.models = []\n        self.histories = []\n\n    def set_random_state(self, random_state):\n        tf.random.set_seed(random_state)\n        np.random.seed(random_state)\n        random.seed(random_state)\n\n    def load_models(self):\n        self.reset()\n\n        indexes = []\n        for index in os.listdir(self.model_path):\n            model_file = os.path.join(self.model_path, index, \"model.h5\")\n            history_file = os.path.join(self.model_path, index, \"history.pkl\")\n            if os.path.exists(model_file) and os.path.exists(history_file):\n                indexes.append(index)\n\n        indexes = np.sort(indexes)\n        for index in indexes:\n            self.models.append(self.load_model(index))\n            self.histories.append(self.load_history(index))\n\n        print(\"loaded\", len(self.models), \"models\")\n\n    def fit(self, *args, **kwargs):\n        model: Model = self.create_model()\n        history = model.fit(*args, **kwargs)\n\n        self.models.append(model)\n        self.histories.append(history.history)\n\n        model_index = len(self.models) - 1\n        self.save_model(model, model_index)\n        self.save_history(history, model_index)\n\n        return history, model\n\n    def create_model(self):\n        return None\n\n    def summary(self, limit=10):\n        min_epochs = np.min([len(history['val_loss']) for history in self.histories])\n\n        val_losses = np.array([history['val_loss'][-min_epochs:] for history in self.histories])\n        val_accuracies = np.array([history['val_acc'][-min_epochs:] for history in self.histories])\n\n        min_val_losses = np.min(val_losses, axis=1)\n        min_val_loss = np.min(min_val_losses)\n        max_val_accuracies = np.max(val_accuracies, axis=1)\n        max_val_accuracy = np.max(max_val_accuracies)\n\n        min_val_loss_mean, min_val_loss_std = np.mean(min_val_losses), np.std(min_val_losses)\n        max_val_accuracy_mean, max_val_accuracy_std = np.mean(max_val_accuracies), np.std(max_val_accuracies)\n\n        min_val_loss_epochs = [np.where(val_loss == np.min(val_loss))[0][0] for val_loss in val_losses]\n        min_val_loss_epoch_mean, min_val_loss_epoch_std = np.mean(min_val_loss_epochs), np.std(min_val_loss_epochs)\n\n        print(\"Model name: {}\".format(self.name))\n        print(\"Description: {}\".format(self.description))\n        print(\"MIN val_loss     = {:.4f}, Mean = {:.4f} Std = {:.4f}\".format(min_val_loss, min_val_loss_mean, min_val_loss_std))\n        print(\"MAX val_accuracy = {:.4f}, Mean = {:.4f} Std = {:.4f}\".format(max_val_accuracy, max_val_accuracy_mean, max_val_accuracy_std))\n        print(\"First epoch of MIN val_loss, Mean = {:.3f} Std = {:.3f}\".format(min_val_loss_epoch_mean, min_val_loss_epoch_std))\n\n    def save_model(self, model, index=None):\n        if index is None or str(index) == \"\":\n            raise ValueError(\"index cannot be empty\")\n\n        model_file = os.path.join(self.model_path, str(index), \"model.h5\")\n        save_model(model, model_file)\n\n    def load_model(self, index):\n        if index is None or str(index) == \"\":\n            raise ValueError(\"index cannot be empty\")\n\n        model_file = os.path.join(self.model_path, str(index), \"model.h5\")\n        return load_model(model_file)\n\n    def save_history(self, history, index=None):\n        if index is None or str(index) == \"\":\n            raise ValueError(\"index cannot be empty\")\n\n        history_file = os.path.join(self.model_path, str(index), \"history.pkl\")\n        hist_df = pd.DataFrame(history.history)\n        hist_df.to_pickle(history_file, protocol=pickle.DEFAULT_PROTOCOL)\n\n    def load_history(self, index):\n        if index is None or str(index) == \"\":\n            raise ValueError(\"index cannot be empty\")\n\n        history_file = os.path.join(self.model_path, str(index), \"history.pkl\")\n        return pd.read_pickle(history_file)\n\n    def create_path(self, file):\n        path = os.path.dirname(file)\n        if not os.path.exists(path):\n            os.makedirs(path, exist_ok=True)\n\n    def plot_images(self, images, label, num_cols=10, width=30, height=20):\n        plt.figure(figsize=(width, height))\n        plt.suptitle(\"label=\" + label)\n\n        num_channels = images.shape[-1]\n        num_rows = (num_channels + 9) // num_cols\n\n        for row in range(num_rows):\n            for col in range(num_cols):\n                channel = row * num_cols + col\n                if channel >= num_channels:\n                    break\n                axs = plt.subplot(num_rows, num_cols, 1 + channel)\n                axs.imshow(images[:, :, channel])\n\n        plt.subplots_adjust(wspace=0.001)\n        plt.show()\n\n    def plot_histories(self, histories, baseline_histories=None):\n        if not isinstance(histories, list):\n            histories = [histories]\n\n        for i in range(len(histories)):\n            # plot loss\n            axs = plt.subplot(2, 1, 1)\n            axs.set_title('Cross Entropy Loss')\n            axs.plot(histories[i]['loss'], color='blue', label='train')\n            axs.plot(histories[i]['val_loss'], color='orange', label='test')\n            # plot accuracy\n            axs = plt.subplot(2, 1, 2)\n            axs.set_title('Classification Accuracy')\n            axs.plot(histories[i]['acc'], color='blue', label='train')\n            axs.plot(histories[i]['val_acc'], color='orange', label='test')\n\n        if baseline_histories is not None:\n            if not isinstance(baseline_histories, list):\n                baseline_histories = [baseline_histories]\n\n            for i in range(len(baseline_histories)):\n                # plot loss\n                axs = plt.subplot(2, 1, 1)\n                axs.set_title('Cross Entropy Loss')\n                axs.plot(baseline_histories[i]['loss'], color='green', label='bl train')\n                axs.plot(baseline_histories[i]['val_loss'], color='red', label='bl test')\n                # plot accuracy\n                axs = plt.subplot(2, 1, 2)\n                axs.set_title('Classification Accuracy')\n                axs.plot(baseline_histories[i]['acc'], color='green', label='bl train')\n                axs.plot(baseline_histories[i]['val_acc'], color='red', label='bl test')\n\n        plt.show()\n\n    # summarize model performance\n    def plot_scores(self, scores):\n        if not isinstance(scores, list):\n            scores = [scores]\n\n        # print summary\n        print('Accuracy: mean=%.3f std=%.3f, n=%d' % (np.mean(scores) * 100, np.std(scores) * 100, len(scores)))\n        # box and whisker plots of results\n        plt.boxplot(scores)\n        plt.show()\n\n    def get_callbacks(self, model_index, reduce_lr=True, tensorboard=False, checkpoint=True, min_loss=None, early_stopping=True):\n        callbacks = []\n        if reduce_lr:\n            reduce_lr_callback = ReduceLROnPlateau(monitor='val_acc',\n                                                   patience=3,\n                                                   verbose=2,\n                                                   factor=.5,\n                                                   min_lr=.00001)\n            callbacks.append(reduce_lr_callback)\n\n        if tensorboard:\n            log_dir = os.path.join(self.model_path, str(model_index), \"logs\")\n            if os.path.exists(log_dir):\n                shutil.rmtree(log_dir, ignore_errors=True)\n            tensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)\n\n            cmd_file = os.path.join(self.model_path, str(model_index), 'tensorboard_start.cmd')\n            self.create_path(cmd_file)\n\n            with open(cmd_file, 'w') as fp:\n                fp.write(\"conda run -n python38 tensorboard --logdir {}\".format(log_dir.replace(\"\\\\\", \"/\")))\n\n            callbacks.append(tensorboard_callback)\n\n        if checkpoint:\n            checkpoint_file = os.path.join(self.model_path, str(model_index), \"model.h5\")\n            callbacks.append(ModelCheckpointV2(filepath=checkpoint_file))\n\n        if early_stopping:\n            callbacks.append(EarlyStoppingV2(patience=10))\n\n        if min_loss is not None:\n            callbacks.append(TrainLossStopping(min_loss))\n\n        return callbacks\n\n\nclass MnistNet(Net):\n    def __init__(self, *args, **kwargs):\n        self.dropout = kwargs.pop('dropout', 0.4)\n        self.input_shape = kwargs.pop('input_shape', (28, 28, 1))\n\n        super(MnistNet, self).__init__(*args, **kwargs)\n\n    def create_model(self):\n        model = Sequential()\n        model.add(InputLayer(input_shape=self.input_shape))\n\n        self.create_layers(model)\n\n        model.add(Dense(10, activation=\"softmax\"))\n        model.compile(loss=\"sparse_categorical_crossentropy\",\n                      optimizer='adam',\n                      metrics=['acc'])\n\n        print(model.summary())\n        return model\n\n    def create_layers(self, model: Model):\n        pass\n\n    def evaluate(self, x_test, y_test, models=None):\n        if models is None:\n            models = self.models\n        elif not isinstance(models, list):\n            models = [models]\n\n        global_preds = []\n        global_accuracies = []\n\n        for model in models:\n            predict = model.predict(x_test)\n            global_preds.append(predict)\n\n            preds = np.argmax(predict, axis=1)\n            num_incorrect = len([index for index, pred in enumerate(preds) if pred != y_test[index]])\n            accuracy = (len(y_test) - num_incorrect) * 1.0 / len(y_test)\n            global_accuracies.append(accuracy)\n\n        global_preds = np.sum(global_preds, axis=0)\n        global_preds = np.argmax(global_preds, axis=1)\n\n        incorrect_indexes = [index for index, pred in enumerate(global_preds) if pred != y_test[index]]\n\n        return global_preds, global_accuracies, incorrect_indexes\n\n    def predict(self, img, real, models=None, title=None):\n        if models is None:\n            models = self.models\n        elif not isinstance(models, list):\n            models = [models]\n\n        corrects = 0\n\n        data = []\n        for index, model in enumerate(models):\n            results = model.predict(np.array([img]))[0]\n            pred = np.argmax(results)\n            data.append([index, pred, real] + list(results))\n\n            if pred == real:\n                corrects += 1\n\n        table = tabulate.tabulate(data, headers=('ID' if title is None else title, 'PRED', 'REAL', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))\n        print(table)\n\n        return corrects\n\n    def predict_submit(self, x_test, models=None, model_path=None):\n        if models is None:\n            models = self.models\n        elif not isinstance(models, list):\n            models = [models]\n\n        model_path = self.model_path if model_path is None else model_path\n        \n        results = [model.predict(x_test) for model in models]\n        results = np.sum(results, axis=0)\n        results = np.argmax(results, axis=1)\n\n        submission_file = os.path.join(model_path, \"submission.csv\")\n        df = pd.DataFrame({'ImageId': range(1, 1 + len(results)), 'Label': results})\n        df.to_csv(submission_file, index=False)\n\n\nclass KaggleNet(MnistNet):\n    def __init__(self, **kwargs):\n        super(KaggleNet, self).__init__(\n            name='kaggle',\n            description='kaggle recommendation',\n            **kwargs)\n\n    def create_layers(self, model: Model):\n        model.add(Conv2D(32, 3, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Conv2D(32, 3, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Conv2D(32, 5, 2, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Dropout(self.dropout))\n\n        model.add(Conv2D(64, 3, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Conv2D(64, 3, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Conv2D(64, 5, 2, activation='relu', padding='same'))\n        model.add(BatchNormalization())\n        model.add(Dropout(self.dropout))\n\n        model.add(Flatten())\n\n        model.add(Dense(128, activation='relu'))\n        model.add(BatchNormalization())\n        model.add(Dropout(self.dropout))\n\n\nclass KaggleNet2(MnistNet):\n    def __init__(self, **kwargs):\n        super(KaggleNet2, self).__init__(\n            name='kaggle2',\n            description='(C*16)K3*12',\n            **kwargs)\n\n    def create_model(self):\n        model = Sequential()\n        model.add(InputLayer(input_shape=self.input_shape))\n\n        # (12-1)*(3-1) + 3 = 25\n        for i in range(12):\n            model.add(Conv2D(16 + 16 * i, 3, use_bias=False))\n            model.add(BatchNormalization())\n            model.add(LeakyReLU())\n        model.add(Dropout(self.dropout))\n\n        model.add(Flatten())\n\n        model.add(Dense(128))\n        model.add(BatchNormalization())\n        model.add(LeakyReLU())\n        model.add(Dropout(self.dropout))\n\n        model.add(Dense(10))\n        model.add(BatchNormalization())\n        model.add(Softmax())\n\n        model.compile(loss=\"sparse_categorical_crossentropy\",\n                      optimizer='adam',\n                      metrics=['acc'])\n\n        print(model.summary())\n        return model\n\nclass KaggleNet3(MnistNet):\n    def __init__(self, **kwargs):\n        super(KaggleNet3, self).__init__(\n            name='kaggle3',\n            description='(C*32)K5*6',\n            **kwargs)\n\n    def create_model(self):\n        model = Sequential()\n        model.add(InputLayer(input_shape=self.input_shape))\n\n        # (6-1)*(5-1) + 5 = 25\n        for i in range(6):\n            model.add(Conv2D(32 + 32 * i, 5, use_bias=False))\n            model.add(BatchNormalization())\n            model.add(LeakyReLU())\n\n        model.add(Dropout(self.dropout))\n        model.add(Flatten())\n\n        model.add(Dense(128))\n        model.add(BatchNormalization())\n        model.add(LeakyReLU())\n        model.add(Dropout(self.dropout))\n\n        model.add(Dense(10))\n        model.add(BatchNormalization())\n        model.add(Softmax())\n\n        model.compile(loss=\"sparse_categorical_crossentropy\",\n                      optimizer='adam',\n                      metrics=['acc'])\n\n        print(model.summary())\n        return model\n\nclass KaggleNet4(MnistNet):\n    def __init__(self, **kwargs):\n        super(KaggleNet4, self).__init__(\n            name='kaggle4',\n            description='(C*48)K7*12',\n            **kwargs)\n\n    def create_model(self):\n        model = Sequential()\n        model.add(InputLayer(input_shape=self.input_shape))\n\n        # 3 * 6 + 7 = 25\n        for i in range(4):\n            model.add(Conv2D(48 + 48 * i, 7, use_bias=False))\n            model.add(BatchNormalization())\n            model.add(LeakyReLU())\n\n        model.add(Dropout(self.dropout))\n        model.add(Flatten())\n\n        model.add(Dense(128))\n        model.add(BatchNormalization())\n        model.add(LeakyReLU())\n        model.add(Dropout(self.dropout))\n\n        model.add(Dense(10))\n        model.add(BatchNormalization())\n        model.add(Softmax())\n\n        model.compile(loss=\"sparse_categorical_crossentropy\",\n                      optimizer='adam',\n                      metrics=['acc'])\n\n        print(model.summary())\n        return model","metadata":{"execution":{"iopub.status.busy":"2022-03-04T06:42:31.623356Z","iopub.execute_input":"2022-03-04T06:42:31.624012Z","iopub.status.idle":"2022-03-04T06:42:31.716311Z","shell.execute_reply.started":"2022-03-04T06:42:31.623969Z","shell.execute_reply":"2022-03-04T06:42:31.715236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mnist = MNIST()\nmnist.load_data_from_local('../input/digit-recognizer')\nmnist.summary()\n\nnet2 = KaggleNet2(model_base=\"./models\")\nnet3 = KaggleNet3(model_base=\"./models\")\nnet4 = KaggleNet4(model_base=\"./models\")\n\nfor net in [net2, net3, net4]:\n    x_train, x_test, y_train, y_test = mnist.random_split(0.2)\n    \n    dgen = ImageDataGenerator(featurewise_center=False,\n                              samplewise_center=False,\n                              featurewise_std_normalization=False,\n                              samplewise_std_normalization=False,\n                              zca_whitening=False,\n                              rotation_range=10,\n                              zoom_range=0.1,\n                              width_shift_range=0.1,\n                              height_shift_range=0.1,\n                              horizontal_flip=False,\n                              vertical_flip=False\n                              )\n\n    net.fit(dgen.flow(x_train, y_train, batch_size=256, shuffle=True),\n            validation_data=(x_test, y_test),\n            epochs=150, verbose=2, callbacks=net.get_callbacks(model_index=0, min_loss=0.002))\n\n    net.summary()","metadata":{"execution":{"iopub.status.busy":"2022-03-04T06:42:31.719370Z","iopub.execute_input":"2022-03-04T06:42:31.719770Z","iopub.status.idle":"2022-03-04T07:51:36.070708Z","shell.execute_reply.started":"2022-03-04T06:42:31.719711Z","shell.execute_reply":"2022-03-04T07:51:36.069958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mnist = MNIST()\nmnist.load_data_from_local('../input/digit-recognizer')\nmnist.summary()\n\nnet2 = KaggleNet2(model_base=\"./models\")\nnet3 = KaggleNet3(model_base=\"./models\")\nnet4 = KaggleNet4(model_base=\"./models\")\n\nnet2.load_models()\nnet3.load_models()\nnet4.load_models()\n    \nnet2.predict_submit(mnist.x_test, [net2.models[0], net3.models[0], net4.models[0]], \"./\")\n\n# public score 0.99657","metadata":{"execution":{"iopub.status.busy":"2022-03-04T07:51:36.072287Z","iopub.execute_input":"2022-03-04T07:51:36.072803Z","iopub.status.idle":"2022-03-04T07:51:46.871605Z","shell.execute_reply.started":"2022-03-04T07:51:36.072763Z","shell.execute_reply":"2022-03-04T07:51:46.870862Z"},"trusted":true},"execution_count":null,"outputs":[]}]}