{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Data Pull & Any functions I make up\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\n\ndef namestr(obj, namespace):\n    return [name for name in namespace if namespace[name] is obj]\n\nimage_dict = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'}\n\n(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\nx_train=x_train.astype(float)\nx_test=x_test.astype(float)\ny_train=y_train.astype(int)\ny_test=y_test.astype(int)\nX_train = x_train/255\nX_test = x_test/255\nsets=[X_train, y_train, X_test, y_test]\nfor i in sets:\n  print('{} shape: {}'.format(namestr(i,globals()), i.shape))\nDNN_X_train=X_train.reshape(X_train.shape[0], 3072)\nDNN_X_test=X_test.reshape(X_test.shape[0], 3072)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-04-25T08:34:41.657182Z","iopub.execute_input":"2022-04-25T08:34:41.657555Z","iopub.status.idle":"2022-04-25T08:34:45.934154Z","shell.execute_reply.started":"2022-04-25T08:34:41.657451Z","shell.execute_reply":"2022-04-25T08:34:45.933229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NOTE: \n## If you are here for the Transformer or the CNN Models, please go to the bottom of the notebook. The best model is the last one!\n\n##### The purpose of this notebook is to show the different outputs of different models and is thus a great reference on how to code various architectures. ","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_train[i].astype(int))\n    plt.xlabel('pred={}'.format(image_dict[y_train[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:34:45.936100Z","iopub.execute_input":"2022-04-25T08:34:45.936471Z","iopub.status.idle":"2022-04-25T08:34:47.445858Z","shell.execute_reply.started":"2022-04-25T08:34:45.936421Z","shell.execute_reply":"2022-04-25T08:34:47.445132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment 1 DNN","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.datasets import mnist\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten\nfrom tensorflow.keras import models\nmodel1 = Sequential()\nmodel1.add(Dense(100, activation='relu', input_shape=[3072]))\nmodel1.add(Dense(10, activation='softmax'))\nmodel1.summary()\nmodel1.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:34:47.446855Z","iopub.execute_input":"2022-04-25T08:34:47.447075Z","iopub.status.idle":"2022-04-25T08:34:48.423452Z","shell.execute_reply.started":"2022-04-25T08:34:47.447047Z","shell.execute_reply":"2022-04-25T08:34:48.422074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history1=model1.fit(DNN_X_train, y_train, batch_size=32, validation_data=(DNN_X_test, y_test), epochs=120, verbose=1)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-04-25T08:34:48.425486Z","iopub.execute_input":"2022-04-25T08:34:48.425825Z","iopub.status.idle":"2022-04-25T08:36:01.789237Z","shell.execute_reply.started":"2022-04-25T08:34:48.425789Z","shell.execute_reply":"2022-04-25T08:36:01.786609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib.pyplot import figure\nplt.figure(figsize=(15,15))\nhistory1_df=pd.DataFrame(history1.history)\nplt.plot(history1_df, linewidth=5)\nplt.legend(history1_df.columns.to_list())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.790009Z","iopub.status.idle":"2022-04-25T08:36:01.790293Z","shell.execute_reply.started":"2022-04-25T08:36:01.790142Z","shell.execute_reply":"2022-04-25T08:36:01.790162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is evidence of overtraining for this DNN even with the little accuracy that it is able to achieve. This is to be expected as it cannot distinguish features from the training vectors and a flat DNN has been proven to be inneffective. Additionally our baseline of 120 epoch proved to only increase the overtraining of the set.","metadata":{}},{"cell_type":"code","source":"y_pred = model1.predict(DNN_X_test)\ny_pred = y_pred.argmax(axis=-1)\nplt.figure(figsize=(20,20))\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_test[i].astype(int))\n    plt.xlabel('pred={}, act={}'.format(image_dict[y_pred[i].item()], image_dict[y_test[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.791918Z","iopub.status.idle":"2022-04-25T08:36:01.792668Z","shell.execute_reply.started":"2022-04-25T08:36:01.792386Z","shell.execute_reply":"2022-04-25T08:36:01.792415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment 2 DNN","metadata":{}},{"cell_type":"code","source":"keras.backend.clear_session()\nmodel2 = Sequential()\nmodel2.add(Dense(200, activation='relu', input_shape=[3072]))\nmodel2.add(Dense(100, activation='relu'))\nmodel2.add(Dense(10, activation='softmax'))\nmodel2.summary()\nmodel2.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.794376Z","iopub.status.idle":"2022-04-25T08:36:01.794978Z","shell.execute_reply.started":"2022-04-25T08:36:01.794727Z","shell.execute_reply":"2022-04-25T08:36:01.794766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history2=model2.fit(DNN_X_train, y_train, batch_size=32, validation_data=(DNN_X_test, y_test), epochs=120, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.796491Z","iopub.status.idle":"2022-04-25T08:36:01.796938Z","shell.execute_reply.started":"2022-04-25T08:36:01.796683Z","shell.execute_reply":"2022-04-25T08:36:01.796707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nhistory2_df=pd.DataFrame(history2.history)\nplt.plot(history2_df, linewidth=5)\nplt.legend(history2_df.columns.to_list())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.799344Z","iopub.status.idle":"2022-04-25T08:36:01.800163Z","shell.execute_reply.started":"2022-04-25T08:36:01.799905Z","shell.execute_reply":"2022-04-25T08:36:01.799931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred2 = model2.predict(DNN_X_test)\ny_pred2 = y_pred2.argmax(axis=-1)\nplt.figure(figsize=(20,20))\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_test[i].astype(int))\n    plt.xlabel('pred={}, act={}'.format(image_dict[y_pred2[i].item()], image_dict[y_test[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.801364Z","iopub.status.idle":"2022-04-25T08:36:01.801983Z","shell.execute_reply.started":"2022-04-25T08:36:01.801696Z","shell.execute_reply":"2022-04-25T08:36:01.801724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment 3 Basic CNN","metadata":{}},{"cell_type":"code","source":"keras.backend.clear_session()\nmodel3=keras.models.Sequential()\nmodel3.add(keras.layers.Conv2D(64,3,3,padding = 'same', activation ='relu', input_shape=(32,32,3)))\nmodel3.add(keras.layers.Conv2D(64,3,3,padding = \"same\", activation = \"relu\"))\n# model3.add(keras.layers.MaxPooling2D(pool_size=(2,2), padding = \"same\"))\nmodel3.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\nmodel3.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\n# model3.add(keras.layers.MaxPooling2D(pool_size=(2,2), padding = \"same\"))\nmodel3.add(keras.layers.Flatten())\nmodel3.add(keras.layers.Dense(128,activation='relu'))\n# model3.add(keras.layers.Dropout(0.5))\nmodel3.add(keras.layers.Dense(10))\nmodel3.add(keras.layers.Activation('softmax'))\nmodel3.summary()\nmodel3.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.803515Z","iopub.status.idle":"2022-04-25T08:36:01.804297Z","shell.execute_reply.started":"2022-04-25T08:36:01.804029Z","shell.execute_reply":"2022-04-25T08:36:01.804058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history3=model3.fit(X_train, y_train, batch_size=32, validation_data=(X_test, y_test), epochs=120, verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.805640Z","iopub.status.idle":"2022-04-25T08:36:01.806626Z","shell.execute_reply.started":"2022-04-25T08:36:01.806352Z","shell.execute_reply":"2022-04-25T08:36:01.806380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nhistory3_df=pd.DataFrame(history3.history)\nplt.plot(history3_df, linewidth=5)\nplt.legend(history3_df.columns.to_list())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.807731Z","iopub.status.idle":"2022-04-25T08:36:01.808523Z","shell.execute_reply.started":"2022-04-25T08:36:01.808266Z","shell.execute_reply":"2022-04-25T08:36:01.808293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This model shows evidence of overtraining. To recuce that we can add dropout layers and kernel normalization.","metadata":{}},{"cell_type":"code","source":"y_pred3 = model3.predict(X_test)\ny_pred3 = y_pred3.argmax(axis=-1)\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_test[i].astype(int))\n    plt.xlabel('pred={}, act={}'.format(image_dict[y_pred3[i].item()], image_dict[y_test[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.811033Z","iopub.status.idle":"2022-04-25T08:36:01.811452Z","shell.execute_reply.started":"2022-04-25T08:36:01.811223Z","shell.execute_reply":"2022-04-25T08:36:01.811247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras.backend.clear_session()\nmodel4=keras.models.Sequential()\nmodel4.add(keras.layers.Conv2D(64,3,3,padding = 'same', activation ='relu', input_shape=(32,32,3)))\nmodel4.add(keras.layers.Conv2D(64,3,3,padding = \"same\", activation = \"relu\"))\nmodel4.add(keras.layers.Dropout(0.1))\nmodel4.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\nmodel4.add(keras.layers.Conv2D(128,3,3,padding = 'same', activation = 'relu'))\nmodel4.add(keras.layers.Dropout(0.2))\nmodel4.add(keras.layers.Conv2D(256,3,3,padding = 'same', activation = 'relu'))\nmodel4.add(keras.layers.Conv2D(256,3,3,padding = 'same', activation = 'relu'))\nmodel4.add(keras.layers.Dropout(0.3))\nmodel4.add(keras.layers.Dense(10))\nmodel4.add(keras.layers.Activation('softmax'))\nmodel4.summary()\nmodel4.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) ","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.812901Z","iopub.status.idle":"2022-04-25T08:36:01.813320Z","shell.execute_reply.started":"2022-04-25T08:36:01.813091Z","shell.execute_reply":"2022-04-25T08:36:01.813114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history4=model4.fit(X_train, y_train, batch_size=32, validation_data=(X_test, y_test), verbose=2)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.814505Z","iopub.status.idle":"2022-04-25T08:36:01.815182Z","shell.execute_reply.started":"2022-04-25T08:36:01.814922Z","shell.execute_reply":"2022-04-25T08:36:01.814948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nhistory4_df=pd.DataFrame(history4.history)\nplt.plot(history4_df, linewidth=5)\nplt.legend(history4_df.columns.to_list())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.816861Z","iopub.status.idle":"2022-04-25T08:36:01.817298Z","shell.execute_reply.started":"2022-04-25T08:36:01.817067Z","shell.execute_reply":"2022-04-25T08:36:01.817090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### upon adding dropout between the convolutional layers we can see our model has improved.","metadata":{}},{"cell_type":"code","source":"y_pred4 = model4.predict(X_test)\ny_pred4 = y_pred4.argmax(axis=-1)\nplt.figure(figsize=(20,20))\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_test[i].astype(int))\n    plt.xlabel('pred={}, act={}'.format(image_dict[y_pred4[i].item()], image_dict[y_test[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.818860Z","iopub.status.idle":"2022-04-25T08:36:01.819288Z","shell.execute_reply.started":"2022-04-25T08:36:01.819058Z","shell.execute_reply":"2022-04-25T08:36:01.819082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### below I will add an example of the weighting of the convolutional layers in the prior model.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import models\nfrom tensorflow.keras.preprocessing import image\nfrom PIL import *\nimport PIL.Image\n(_,_), (test_images, test_labels) = tf.keras.datasets.cifar10.load_data()\n\nimg = test_images[100]\nimg_tensor = image.img_to_array(img)\nimg_tensor = np.expand_dims(img_tensor, axis=0)\n\nclass_names = ['airplane'\n,'automobile'\n,'bird'\n,'cat'\n,'deer'\n,'dog'\n,'frog' \n,'horse'\n,'ship'\n,'truck']\n\nplt.imshow(img, cmap='viridis')\nplt.axis('off')\nplt.show()\n\nlayer_outputs = [layer.output for layer in model4.layers[:8]]\n# Creates a model that will return these outputs, given the model input:\nactivation_model = models.Model(inputs=model4.input, outputs=layer_outputs)\nactivations = activation_model.predict(img_tensor)\nlayer_names = []\nfor layer in model4.layers[:3]:\n    layer_names.append(layer.name)\n\nimages_per_row = 16\n\n# Now let's display our feature maps\nfor layer_name, layer_activation in zip(layer_names, activations):\n    # This is the number of features in the feature map\n    n_features = layer_activation.shape[-1]\n\n    # The feature map has shape (1, size, size, n_features)\n    size = layer_activation.shape[1]\n\n    # We will tile the activation channels in this matrix\n    n_cols = n_features // images_per_row\n    display_grid = np.zeros((size * n_cols, images_per_row * size))\n\n    # We'll tile each filter into this big horizontal grid\n    for col in range(n_cols):\n        for row in range(images_per_row):\n            channel_image = layer_activation[0,\n                                             :, :,\n                                             col * images_per_row + row]\n            # Post-process the feature to make it visually palatable\n            channel_image -= channel_image.mean()\n            channel_image /= channel_image.std()\n            channel_image *= 64\n            channel_image += 128\n            channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n            display_grid[col * size : (col + 1) * size,\n                         row * size : (row + 1) * size] = channel_image\n\n    # Display the grid\n    scale = 1. / size\n    plt.figure(figsize=(scale * display_grid.shape[1],\n                        scale * display_grid.shape[0]))\n    plt.title(layer_name)\n    plt.grid(False)\n    plt.imshow(display_grid, aspect='auto', cmap='viridis')\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.820811Z","iopub.status.idle":"2022-04-25T08:36:01.821227Z","shell.execute_reply.started":"2022-04-25T08:36:01.820999Z","shell.execute_reply":"2022-04-25T08:36:01.821022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Experiment 5 Vision Transformer","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\n\nwd = 0.0001\nlr = 0.001\nbatch = 128\nepochs = 120\nimage_size = 72\npatch_size = 4\npatch_dist = 18**2\nprojection = 64\nnum_heads = 4\ntransformer_val = [128, 64]\nlayers = 8\nmlp_head_units = [2048, 1024]\ndata_augmentation = keras.Sequential(\n    [\n        keras.layers.experimental.preprocessing.Normalization(),\n        keras.layers.experimental.preprocessing.Resizing(image_size, image_size),\n        keras.layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        keras.layers.experimental.preprocessing.RandomRotation(factor=0.02),\n        keras.layers.experimental.preprocessing.RandomZoom(\n            height_factor=0.2, width_factor=0.2\n        ),\n    ],\n    name=\"data_augmentation\",\n)\ndata_augmentation.layers[0].adapt(x_train)\ndef mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = keras.layers.Dense(units, activation=tf.nn.gelu)(x)\n        x = keras.layers.Dropout(dropout_rate)(x)\n    return x\n\nclass Patches(keras.layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n\n    def call(self, images):\n        batch = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images=images,\n            sizes=[1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        patches = tf.reshape(patches, [batch, -1, patch_dims])\n        return patches\nplt.figure(figsize=(4, 4))\nimage = x_train[np.random.choice(range(x_train.shape[0]))]\nplt.imshow(image.astype(\"uint8\"))\nplt.axis(\"off\")\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\nn = int(np.sqrt(patches.shape[1]))\nplt.figure(figsize=(4, 4))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 3))\n    plt.imshow(patch_img.numpy().astype(\"uint8\"))\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.822855Z","iopub.status.idle":"2022-04-25T08:36:01.823282Z","shell.execute_reply.started":"2022-04-25T08:36:01.823053Z","shell.execute_reply":"2022-04-25T08:36:01.823077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEncoder(keras.layers.Layer):\n    def __init__(self, patch_dist, projection):\n        super(PatchEncoder, self).__init__()\n        self.patch_dist = patch_dist\n        self.projection = keras.layers.Dense(units=projection)\n        self.position_embedding = keras.layers.Embedding(\n            input_dim=patch_dist, output_dim=projection\n        )\n\n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.patch_dist, delta=1)\n        encoded = self.projection(patch) + self.position_embedding(positions)\n        return encoded\n\ndef create_vit_classifier():\n    inputs = keras.layers.Input(shape=(32, 32, 3))\n    augmented = data_augmentation(inputs)\n    patches = Patches(patch_size)(augmented)\n    encoded_patches = PatchEncoder(patch_dist, projection)(patches)\n\n    for _ in range(layers):\n        x1 = keras.layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n        attention_output = keras.layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection, dropout=0.1\n        )(x1, x1)\n        x2 = keras.layers.Add()([attention_output, encoded_patches])\n        x3 = keras.layers.LayerNormalization(epsilon=1e-6)(x2)\n        x3 = mlp(x3, hidden_units=transformer_val, dropout_rate=0.1)\n        encoded_patches = keras.layers.Add()([x3, x2])\n    representation = keras.layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n    representation = keras.layers.Flatten()(representation)\n    representation = keras.layers.Dropout(0.5)(representation)\n    features = mlp(representation, hidden_units=mlp_head_units, dropout_rate=0.5)\n    logits = keras.layers.Dense(10)(features)\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model\n\ndef run_experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        lr=lr, weight_decay=wd)\n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.SparseCategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.SparseTopKCategoricalAccuracy(5, name=\"top-5-accuracy\"),],)\n    checkpoint_filepath = \"/tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,)\n    history = model.fit(\n        x=x_train,\n        y=y_train,\n        batch_size=batch,\n        epochs=epochs,\n        validation_split=0.1,\n        callbacks=[checkpoint_callback],)\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, top_5_accuracy = model.evaluate(x_test, y_test)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test top 5 accuracy: {round(top_5_accuracy * 100, 2)}%\")\n    return history\n\nvit_classifier = create_vit_classifier()\nhistory6 = run_experiment(vit_classifier)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.824907Z","iopub.status.idle":"2022-04-25T08:36:01.825328Z","shell.execute_reply.started":"2022-04-25T08:36:01.825097Z","shell.execute_reply":"2022-04-25T08:36:01.825121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred=pd.DataFrame(vit_classifier.predict(x_test))\ny_pred['y_pred'] = y_pred.idxmax(axis=1)\ny_pred['true_value']=y_test","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.827002Z","iopub.status.idle":"2022-04-25T08:36:01.827437Z","shell.execute_reply.started":"2022-04-25T08:36:01.827206Z","shell.execute_reply":"2022-04-25T08:36:01.827229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = y_pred['y_pred'].to_numpy()\nplt.figure(figsize=(20,20))\nfor i in range(16):\n    plt.subplot(4,4, i+1)\n    plt.imshow(x_test[i].astype(int))\n    plt.xlabel('pred={}, act={}'.format(image_dict[y_pred[i].item()], image_dict[y_test[i].item()]), fontsize=15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.828675Z","iopub.status.idle":"2022-04-25T08:36:01.829607Z","shell.execute_reply.started":"2022-04-25T08:36:01.829342Z","shell.execute_reply":"2022-04-25T08:36:01.829370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### It seems like the patches were too small in the prior model to pick up the necessary features. However, this shows the improvement a Transformer makes on image classification over a CNN.","metadata":{}},{"cell_type":"markdown","source":"### Final Experiment\n\n##### We have to redefine all functionas as the patch size was hard coded in my prior example.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import layers\nimport tensorflow_addons as tfa\nimport matplotlib.pyplot as plt\n\nwd = 0.0001\nlr = 0.001\nbatch = 256\nepochs = 80\nimage_size = 72\npatch_size = 8\npatch_dist = 9**2\nprojection = 64\nnum_heads = 4\ntransformer_val = [128, 64]\nlayers = 8\nmlp_head_units = [2048, 1024]\ndata_augmentation = keras.Sequential(\n    [\n        keras.layers.experimental.preprocessing.Normalization(),\n        keras.layers.experimental.preprocessing.Resizing(image_size, image_size),\n        keras.layers.experimental.preprocessing.RandomFlip(\"horizontal\"),\n        keras.layers.experimental.preprocessing.RandomRotation(factor=0.02),\n        keras.layers.experimental.preprocessing.RandomZoom(\n            height_factor=0.2, width_factor=0.2\n        ),\n    ],\n    name=\"data_augmentation\",\n)\ndata_augmentation.layers[0].adapt(x_train)\ndef mlp(x, hidden_units, dropout_rate):\n    for units in hidden_units:\n        x = keras.layers.Dense(units, activation=tf.nn.gelu)(x)\n        x = keras.layers.Dropout(dropout_rate)(x)\n    return x\n\nclass Patches(keras.layers.Layer):\n    def __init__(self, patch_size):\n        super(Patches, self).__init__()\n        self.patch_size = patch_size\n\n    def call(self, images):\n        batch = tf.shape(images)[0]\n        patches = tf.image.extract_patches(\n            images=images,\n            sizes=[1, self.patch_size, self.patch_size, 1],\n            strides=[1, self.patch_size, self.patch_size, 1],\n            rates=[1, 1, 1, 1],\n            padding=\"VALID\",\n        )\n        patch_dims = patches.shape[-1]\n        patches = tf.reshape(patches, [batch, -1, patch_dims])\n        return patches\nplt.figure(figsize=(4, 4))\nimage = x_train[np.random.choice(range(x_train.shape[0]))]\nplt.imshow(image.astype(\"uint8\"))\nplt.axis(\"off\")\nresized_image = tf.image.resize(\n    tf.convert_to_tensor([image]), size=(image_size, image_size)\n)\npatches = Patches(patch_size)(resized_image)\nprint(f\"Image size: {image_size} X {image_size}\")\nprint(f\"Patch size: {patch_size} X {patch_size}\")\nprint(f\"Patches per image: {patches.shape[1]}\")\nprint(f\"Elements per patch: {patches.shape[-1]}\")\nn = int(np.sqrt(patches.shape[1]))\nplt.figure(figsize=(4, 4))\nfor i, patch in enumerate(patches[0]):\n    ax = plt.subplot(n, n, i + 1)\n    patch_img = tf.reshape(patch, (patch_size, patch_size, 3))\n    plt.imshow(patch_img.numpy().astype(\"uint8\"))\n    plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.831008Z","iopub.status.idle":"2022-04-25T08:36:01.831434Z","shell.execute_reply.started":"2022-04-25T08:36:01.831202Z","shell.execute_reply":"2022-04-25T08:36:01.831226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PatchEncoder(keras.layers.Layer):\n    def __init__(self, patch_dist, projection):\n        super(PatchEncoder, self).__init__()\n        self.patch_dist = patch_dist\n        self.projection = keras.layers.Dense(units=projection)\n        self.position_embedding = keras.layers.Embedding(\n            input_dim=patch_dist, output_dim=projection\n        )\n\n    def call(self, patch):\n        positions = tf.range(start=0, limit=self.patch_dist, delta=1)\n        encoded = self.projection(patch) + self.position_embedding(positions)\n        return encoded\n\ndef create_vit_classifier():\n    inputs = keras.layers.Input(shape=(32, 32, 3))\n    augmented = data_augmentation(inputs)\n    patches = Patches(patch_size)(augmented)\n    encoded_patches = PatchEncoder(patch_dist, projection)(patches)\n\n    for _ in range(layers):\n        x1 = keras.layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n        attention_output = keras.layers.MultiHeadAttention(\n            num_heads=num_heads, key_dim=projection, dropout=0.1\n        )(x1, x1)\n        x2 = keras.layers.Add()([attention_output, encoded_patches])\n        x3 = keras.layers.LayerNormalization(epsilon=1e-6)(x2)\n        x3 = mlp(x3, hidden_units=transformer_val, dropout_rate=0.1)\n        encoded_patches = keras.layers.Add()([x3, x2])\n    representation = keras.layers.LayerNormalization(epsilon=1e-6)(encoded_patches)\n    representation = keras.layers.Flatten()(representation)\n    representation = keras.layers.Dropout(0.5)(representation)\n    features = mlp(representation, hidden_units=mlp_head_units, dropout_rate=0.5)\n    logits = keras.layers.Dense(10)(features)\n    model = keras.Model(inputs=inputs, outputs=logits)\n    return model\n\ndef run_experiment(model):\n    optimizer = tfa.optimizers.AdamW(\n        lr=lr, weight_decay=wd)\n    model.compile(\n        optimizer=optimizer,\n        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n        metrics=[\n            keras.metrics.SparseCategoricalAccuracy(name=\"accuracy\"),\n            keras.metrics.SparseTopKCategoricalAccuracy(5, name=\"top-5-accuracy\"),],)\n    checkpoint_filepath = \"/tmp/checkpoint\"\n    checkpoint_callback = keras.callbacks.ModelCheckpoint(\n        checkpoint_filepath,\n        monitor=\"val_accuracy\",\n        save_best_only=True,\n        save_weights_only=True,)\n    history = model.fit(\n        x=x_train,\n        y=y_train,\n        batch_size=batch,\n        epochs=epochs,\n        validation_split=0.1,\n        callbacks=[checkpoint_callback],)\n    model.load_weights(checkpoint_filepath)\n    _, accuracy, top_5_accuracy = model.evaluate(x_test, y_test)\n    print(f\"Test accuracy: {round(accuracy * 100, 2)}%\")\n    print(f\"Test top 5 accuracy: {round(top_5_accuracy * 100, 2)}%\")\n    return history\n\nvit_classifier = create_vit_classifier()\nhistory6 = run_experiment(vit_classifier)","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.832999Z","iopub.status.idle":"2022-04-25T08:36:01.833420Z","shell.execute_reply.started":"2022-04-25T08:36:01.833192Z","shell.execute_reply":"2022-04-25T08:36:01.833215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nhistory6_df=pd.DataFrame(history6.history)\nplt.plot(history6_df, linewidth=5)\nplt.legend(history6_df.columns.to_list())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-25T08:36:01.834879Z","iopub.status.idle":"2022-04-25T08:36:01.835310Z","shell.execute_reply.started":"2022-04-25T08:36:01.835074Z","shell.execute_reply":"2022-04-25T08:36:01.835099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}