{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from dataclasses import dataclass\nimport math\nfrom pathlib import Path\nimport tensorflow as tf\nfrom tensorflow.data import AUTOTUNE, Dataset\nfrom tensorflow.keras.initializers import HeNormal\nfrom tensorflow.keras.layers import Input\nfrom tensorflow.keras.layers import Conv2D, Dense, BatchNormalization, Activation, Dropout\nfrom tensorflow.keras.layers import RandomZoom, RandomFlip, RandomTranslation, RandomRotation\nfrom tensorflow.keras.models import Model\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-21T18:00:40.382637Z","iopub.execute_input":"2022-01-21T18:00:40.382891Z","iopub.status.idle":"2022-01-21T18:00:45.366709Z","shell.execute_reply.started":"2022-01-21T18:00:40.382818Z","shell.execute_reply":"2022-01-21T18:00:45.365981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR=Path('../input/digit-recognizer/')\n\nBATCH_SIZE=32","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:25:13.1414Z","iopub.execute_input":"2022-01-21T15:25:13.141806Z","iopub.status.idle":"2022-01-21T15:25:13.145877Z","shell.execute_reply.started":"2022-01-21T15:25:13.141748Z","shell.execute_reply":"2022-01-21T15:25:13.144823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(INPUT_DIR / 'train.csv')\ntest = pd.read_csv(INPUT_DIR / 'test.csv')\nY_train = np.array(train.label.values.astype('int32'))\nX_train = np.array((train.iloc[:,1:].values).astype('float32')) / 255.\nX_test = np.array((test.values).astype('float32')) / 255.\nprint(f\"{np.mean(X_train)} {np.std(X_train)}\")\nprint(Y_train[:10])\n\nNUM_CLASSES = np.max(Y_train)+1\nINPUT_SHAPE_2D = (28,28)\nINPUT_SHAPE = (*INPUT_SHAPE_2D, 1)\nPIXEL_MEAN = np.mean(X_train)\nPIXEL_STD = np.std(X_train)\n\ndef one_hot(data, classes=NUM_CLASSES):\n    eye = np.eye(classes)\n    return eye[data]\n\ndef normalize(pixels):\n    return (pixels - PIXEL_MEAN) / PIXEL_STD\n\ndef denormalize(pixels):\n    return pixels * PIXEL_STD + PIXEL_MEAN\n\nX_train = normalize(X_train).reshape((-1, *INPUT_SHAPE_2D, 1))\nX_test = normalize(X_test).reshape((-1, *INPUT_SHAPE_2D, 1))\nY_train = one_hot(Y_train)\nprint(Y_train[:10])","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:25:16.225759Z","iopub.execute_input":"2022-01-21T15:25:16.226321Z","iopub.status.idle":"2022-01-21T15:25:21.586238Z","shell.execute_reply.started":"2022-01-21T15:25:16.226282Z","shell.execute_reply":"2022-01-21T15:25:21.5848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create dataset from csv tensors\nfull_set = Dataset.from_tensor_slices((X_train, Y_train))\n\ndef make_train_valid(dataset, split=(0.9,)):\n    ds_len = len(dataset)\n    train_size = math.floor(split[0] * ds_len)\n    test_size = ds_len - train_size\n    \n    ds = dataset.shuffle(len(dataset), reshuffle_each_iteration=False)\n    train = ds.take(train_size)\n    valid = ds.skip(train_size).take(test_size)\n    \n    # TODO: Add test option as well.\n    return train, valid\n\nimport logging\ndef plot_images(batch, labels=None, images_per_side=2):\n    logger = logging.getLogger()\n    old_level = logger.level\n    logger.setLevel(100)\n    \n    fig = plt.figure(figsize=(3*images_per_side,3*images_per_side))\n    num_images = images_per_side ** 2\n    \n    images = batch\n    for i in range(num_images):\n        f = fig.add_subplot(images_per_side,images_per_side,i+1)\n        im = denormalize(images[i])\n        im = np.reshape(im, INPUT_SHAPE_2D)\n\n        # Pull batch[1] for the title, if labels provided, then assume one-hot encoded\n        # label that is mapped to a string by the labels input... not very extensible\n        # but okay for now.\n        if labels is not None:\n            title = labels[i]\n            f.set_title(title, color='black')\n            \n        f.set_xticklabels([])\n        f.set_xticks([])\n        f.set_yticklabels([])\n        f.set_yticks([])\n        plt.imshow(im)\n    logger.setLevel(old_level)","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:25:21.58789Z","iopub.execute_input":"2022-01-21T15:25:21.588696Z","iopub.status.idle":"2022-01-21T15:25:24.106077Z","shell.execute_reply.started":"2022-01-21T15:25:21.588655Z","shell.execute_reply":"2022-01-21T15:25:24.105351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_augmentation = tf.keras.Sequential([\n    tf.keras.layers.RandomRotation(0.1, fill_mode='nearest'),\n    tf.keras.layers.RandomZoom(0.1, fill_mode='nearest'),\n    tf.keras.layers.RandomTranslation(0.1, 0.1, fill_mode='nearest'),\n    tf.keras.layers.RandomContrast(0.1)\n])\n\ntrain, valid = make_train_valid(full_set)\ntrain = train.shuffle(len(train))\\\n    .batch(BATCH_SIZE)\\\n    .map(lambda x, y: (data_augmentation(x, training=True), y),\n         num_parallel_calls=AUTOTUNE)\\\n    .prefetch(buffer_size=AUTOTUNE)\nvalid = valid.batch(BATCH_SIZE)\\\n    .prefetch(buffer_size=AUTOTUNE)\n\nbatch = next(iter(train))\nprint(batch[0].shape)\nplot_images(batch[0], np.argmax(batch[1], axis=1), images_per_side=4)\n\ntest = Dataset.from_tensor_slices(X_test)\\\n    .batch(BATCH_SIZE)\\\n    .prefetch(buffer_size=AUTOTUNE)\nbatch_test = next(iter(test))\nplot_images(batch_test)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:25:49.927574Z","iopub.execute_input":"2022-01-21T15:25:49.927863Z","iopub.status.idle":"2022-01-21T15:25:52.390277Z","shell.execute_reply.started":"2022-01-21T15:25:49.927833Z","shell.execute_reply":"2022-01-21T15:25:52.389591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ModelSaver(tf.keras.callbacks.Callback):\n    def __init__(self, model, model_dir=None, model_name=None):\n        self.__model = model\n        self.__eps = 0.0002\n        \n        self.__model_dir = model_dir\n        self.__save_name(model_name)\n\n        self.__saved_models = {}\n        self.__best_score = float('-inf')\n        \n    def __save_name(self, model_name):\n        if model_name is None:\n            self.__base_name = self.model.name\n        else:\n            self.__base_name = model_name \n            \n    def __model_path(self):\n        p = Path(self.__model_dir)\n        model_file = p / f'{self.__base_name}.h5'\n        return model_file\n        \n    def save_model(self, score):\n        if self.__model_dir is None:\n            return\n        \n        model_file = self.__model_path()\n        self.__model.save(model_file, save_format='h5')\n        self.__saved_models[model_file] = score\n        \n    def saved_model_info(self):\n        return next(iter(self.__saved_models.items()))\n                 \n    def load_best_model(self):\n        # TODO: update to find the lowest scoring __saved_models file\n        return tf.keras.models.load_model(self.__model_path())\n\n    def on_epoch_end(self, epoch, logs=None):\n        score = logs['val_categorical_accuracy']\n        if score > (self.__best_score + self.__eps):\n            self.__best_score = score\n            self.save_model(score)\n            print(f'Saved Model with score {score}')\n\n@dataclass\nclass layer_config:\n    layer_num: int\n    filters: int\n    stride: int\n    dropout: float\n    skip_ahead: bool  \n    \nMODEL_CONFIG = [\n    layer_config(0, 25, 2, 0.33, False),\n    layer_config(1, 50, 1, 0.5, False),\n    layer_config(2, 200, 1, 0.25, True),\n    layer_config(3, 400, 2, 0.5, False),\n    layer_config(4, 250, 1, 0.1, False),\n    layer_config(5, 50, 1, 0.6, True),\n    layer_config(6, 325, 1, 0.3, False),\n]\n\ndef conv_block(config, x):\n    # With striding, there may be issues with invalid Conv configurations...\n    # Let's avoid this for now by doing 'same' padding which makes it a non-issue,\n    # but worth testing to see what happens if an invalid configuration is hit.\n    x = Conv2D(config.filters, 3,\n               strides=config.stride,\n               padding='same',\n               activation='relu',\n               kernel_initializer=HeNormal(),\n               name=f'conv2d-{config.layer_num}')(x)\n    x = BatchNormalization(name=f'bn-{config.layer_num}')(x)\n    x = Dropout(config.dropout, name=f'drop-{config.layer_num}')(x)\n    return x\n\ndef create_image_model(input_shape=INPUT_SHAPE, n_classes=NUM_CLASSES):\n    inp = Input(input_shape)\n    x = inp\n    \n    skip_layers = []\n    for config in MODEL_CONFIG:\n        x = conv_block(config, x)\n        if config.skip_ahead:\n            skip_layers.append(x)  # Add skip-ahead to the final result -- similar to densenet blocks\n   \n    pools = []\n    pools.append(tf.keras.layers.GlobalAveragePooling2D()(x))\n    for skip in skip_layers:\n        pools.append(tf.keras.layers.GlobalAveragePooling2D()(skip))\n    x = tf.concat(pools, axis=-1)\n    x = Dropout(0.6)(x)\n    \n    x = Dense(n_classes)(x)\n\n    return Model(inp, x)\n\ndef create_model(input_shape=INPUT_SHAPE, n_classes=NUM_CLASSES):\n    inp = Input(input_shape)\n    x = inp\n    \n    image_model = create_image_model(input_shape, n_classes)\n    print(image_model.summary())\n    x = image_model(x)\n    \n    return Model(inp, x), image_model\n\ndef compile_model(model):\n    loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n    opt = tf.keras.optimizers.Adam()\n    accuracy = tf.keras.metrics.CategoricalAccuracy()\n    top2_accuracy = tf.keras.metrics.TopKCategoricalAccuracy(k=2)\n    model.compile(loss=loss, optimizer=opt, metrics=[accuracy, top2_accuracy])\n\nfull_model, model = create_model()\ncompile_model(full_model)\n\nmodel_saver = ModelSaver(model, 'saved_models', 'baseline')\nprint(\"Running Training\")\nfull_model.fit(train,\n               validation_data=valid,\n               epochs=100,\n               callbacks=[model_saver,\n                          tf.keras.callbacks.EarlyStopping(monitor='val_categorical_accuracy', patience=10)])\n\nsubmission_model = model\n","metadata":{"execution":{"iopub.status.busy":"2022-01-21T15:27:56.725525Z","iopub.execute_input":"2022-01-21T15:27:56.725818Z","iopub.status.idle":"2022-01-21T15:44:44.061358Z","shell.execute_reply.started":"2022-01-21T15:27:56.725767Z","shell.execute_reply":"2022-01-21T15:44:44.060629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_submission(model, dataset=X_test):\n    res = model.predict(dataset)\n    labels = np.argmax(res, axis=1)\n    index = np.array([x for x in range(1, len(labels)+1)])\n    print(index.shape)\n    print(labels.shape)\n    result = pd.DataFrame({'ImageId': index, 'Label': labels})\n    print(result.head(5))\n    result.to_csv('submission.csv', index=False)\n    return result\n\nt = next(iter(test))\nsub = create_submission(submission_model, dataset=test)\nplot_images(t, sub['Label'], images_per_side=4)","metadata":{"execution":{"iopub.status.busy":"2022-01-19T12:19:56.835643Z","iopub.execute_input":"2022-01-19T12:19:56.835915Z","iopub.status.idle":"2022-01-19T12:21:11.331642Z","shell.execute_reply.started":"2022-01-19T12:19:56.835883Z","shell.execute_reply":"2022-01-19T12:21:11.330933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}