{"cells":[{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","id":"FixzSGnQmoac","outputId":"1f3ab035-7303-4913-dd10-950852d4c2f2","trusted":true,"_uuid":"013249e67e1152b22d71c3fe8db4ec55e1bfb643"},"cell_type":"code","source":"%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\n\nfrom time import time\nfrom collections import Counter\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# My seed\n\nseed = 42","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"CKPsrUUlQl5j","trusted":true,"_uuid":"d5e52270d41ceee7b2ccde3f9597cf62d3841a4c"},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"K9D2H1wJRLrR","_uuid":"76284c63c515f0d2da81fb9c812d6ad505b490a2"},"cell_type":"markdown","source":"### Splitting the dataset"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":52},"colab_type":"code","id":"L1ShW20EQr0M","outputId":"ddf1b6da-06d8-488e-b47a-3ef1c231124a","trusted":true,"_uuid":"bfd4233ab5d1dfe4de6e585535a645d9b06cfeeb"},"cell_type":"code","source":"X_train = df_train.drop(['label'], axis=1)\ny_train = df_train['label']\n\n# Free memory space\n\ndel df_train\n\nprint('Shape of X_train:', X_train.shape)\nprint('Shape of y_train:', y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"6j6yJtj5SIuK","_uuid":"6d4eff51394471109f31205c62f5b4168b0cebfa"},"cell_type":"markdown","source":"### Normalizing the values of training and test"},{"metadata":{"colab":{},"colab_type":"code","id":"SfkmFge5SJHt","trusted":true,"_uuid":"81134a2002d2d244c873ec3a7950c333fc08a26c"},"cell_type":"code","source":"X_train = X_train / 255","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"mODMR1dLSR9t","_uuid":"9c97475ab4bf94fa498a64be1a2b18cde995c446"},"cell_type":"markdown","source":"### Reshape the images in 4 dimensions to use with Keras"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":34},"colab_type":"code","id":"d9fq_86pSSiy","outputId":"412f9fa9-ad20-4a91-fbe4-33cddf035cae","trusted":true,"_uuid":"abf950aab484be888fe283b63e614e07b69e84e7"},"cell_type":"code","source":"X_train = X_train.values.reshape(-1,28,28,1) # (height = 28px, width = 28px , canal = 1)\nprint('Shape of X_train:', X_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"7X5vx9qOSdEF","_uuid":"3531e8fa932441468a815044f62b424f3d768360"},"cell_type":"markdown","source":"### Converting y values (labels) to categorical values"},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":139},"colab_type":"code","id":"KW4URT0SSddz","outputId":"9c1d1fe3-9ff0-41da-de55-53e8e373ab8e","trusted":true,"_uuid":"b98a8527b7dce0c6716a2312af4e8d1c0e4fe735"},"cell_type":"code","source":"# One Hot Categories\n\ny_train = to_categorical(y_train, num_classes = 10)\ny_train","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"-4-yC1WnS0WB","_uuid":"e413d74d07728791f00716d12af6216c678da9c4"},"cell_type":"markdown","source":"### Function to create neural networks to be evalueted"},{"metadata":{"colab":{},"colab_type":"code","id":"kPTN6fqsTAYb","trusted":true,"_uuid":"cb9a65c77cc788d60c4b5f8e158e2f5929cbf816"},"cell_type":"code","source":"def baseline_model(layers = 1, \n                   filter_l1 = 32, \n                   filter_l2 = 64, \n                   filter_l3 = 128,\n                   kernel_size_l1 = 3,\n                   kernel_size_l2 = 3,\n                   kernel_size_l3 = 3,\n                   pool_size_l1 = 2,\n                   pool_size_l2 = 2,\n                   pool_size_l3 = 2,\n                   activation_l1 = 'relu',\n                   activation_l2 = 'relu',\n                   activation_l3 = 'relu',\n                   optimizer = 'Adamax',\n                   dense_1 = 256,\n                   dense_2 = 256,\n                   dropout_1 = 0.25, \n                   dropout_2 = 0.25, \n                   dropout_3 = 0.4):\n    \n    # Create baseline\n    \n    model = Sequential()\n\n    # First group\n    #--------------------------------------------------\n    model.add(Conv2D(filter_l1, (kernel_size_l1, kernel_size_l1), \n                     padding='same', activation=activation_l1,\n                    input_shape = (28, 28, 1)))\n    \n    if (layers >= 2):\n        for i in range(layers-1):\n            model.add(Conv2D(filter_l1, (kernel_size_l1, kernel_size_l1), \n                             padding='same', activation=activation_l1))\n        \n    model.add(MaxPool2D(pool_size=(pool_size_l1, pool_size_l1)))\n    \n    # Second group\n    #--------------------------------------------------\n    model.add(Conv2D(filter_l2, (kernel_size_l2, kernel_size_l2), \n                     padding='same', activation=activation_l2))\n    \n    if (layers >= 2):\n        for i in range(layers-1):\n            model.add(Conv2D(filter_l2, (kernel_size_l2, kernel_size_l2), \n                             padding='same', activation=activation_l2))\n        \n    model.add(MaxPool2D(pool_size=(pool_size_l2, pool_size_l2)))\n    \n    # Third group\n    #--------------------------------------------------\n    model.add(Conv2D(filter_l3, (kernel_size_l3, kernel_size_l3), \n                     padding='same', activation=activation_l3))\n    \n    if (layers >= 2):\n        for i in range(layers-1):\n            model.add(Conv2D(filter_l3, (kernel_size_l3, kernel_size_l3),\n                             padding='same', activation=activation_l3))\n        \n    model.add(MaxPool2D(pool_size=(pool_size_l3, pool_size_l3)))\n    \n    #--------------------------------------------------\n    model.add(Dropout(dropout_1))\n    model.add(Flatten())\n              \n    model.add(Dense(output_dim=dense_1, activation='relu'))\n    model.add(Dropout(dropout_2))\n              \n    model.add(Dense(output_dim=dense_2, activation='relu'))\n    model.add(Dropout(dropout_3))\n    \n    #--------------------------------------------------\n    model.add(Dense(10, activation = \"softmax\"))\n    \n    # Compile the baseline model including the optimizer and evaluating \n    # the performance of the baseline by accuracy\n    \n    model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63193f90d9dd28f4b5c15e7630e6dd24d7485f69"},"cell_type":"markdown","source":"### Learning rate"},{"metadata":{"trusted":true,"_uuid":"95e772933f127900e55307d4a82d291d3e93a5c5"},"cell_type":"code","source":"# If after the third epoch we didn't have an improvement of validation accuracy, \n# the learning rate will be decreased by 50% (factor).\n\nlr_reduction = ReduceLROnPlateau(monitor='val_acc',\n                                 patience=3, \n                                 verbose=0, \n                                 factor=0.5, \n                                 min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f7c47e7474611a6f2ff1c381aab4b33e27c077a"},"cell_type":"markdown","source":"### Data augmentation"},{"metadata":{"trusted":true,"_uuid":"5b3fec28b7e8bed6463edeba7166c951bf03a8c0"},"cell_type":"code","source":"# The idea is to alter the training data with small transformations to reproduce the variations \n# occuring when someone is writing a digit. It's a way to minimize the overfitting of the model.\n\ngenerator = ImageDataGenerator(featurewise_center = False,\n                               samplewise_center = False, \n                               featurewise_std_normalization = False,\n                               samplewise_std_normalization = False,\n                               zca_whitening = False,\n                               rotation_range = 10, # Rotate image in 10 degrees\n                               zoom_range = 0.10, # Zoom image (10% zoom) \n                               width_shift_range = 0.10, # Shift image horizontally (10% of width)\n                               height_shift_range = 0.10, # Shift image vertically (10% of height)\n                               horizontal_flip = False,\n                               vertical_flip = False)","execution_count":null,"outputs":[]},{"metadata":{"colab_type":"text","id":"D_jbc38cZ4AR","_uuid":"5e1debf863a54e929cadbbcd3437832985299242"},"cell_type":"markdown","source":"### Testing the models"},{"metadata":{"colab":{},"colab_type":"code","id":"KFziKX3UZ5ma","trusted":true,"_uuid":"637ed81fbd49e6a4a0fb80ba955a6c3ee09680b0"},"cell_type":"code","source":"# I tried to use 5 epochs, but sometimes I didn't get to the finish line. \n\nepochs = 5\nbatch_size = 90\n\n# Dictionary of parameters that gave the best results\n\nbest = {}","execution_count":null,"outputs":[]},{"metadata":{"colab":{},"colab_type":"code","id":"SiXoRqYrf0sO","trusted":true,"_uuid":"ec46a6e818ab8248f9d246a8b9fc399dfafadec4"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":347},"colab_type":"code","id":"g_9nrr63aIed","outputId":"58a8eca6-69bc-47f7-f12c-66cda1ce4346","trusted":true,"_uuid":"df413063a6c3d293645b8a136efa07551f05bcba"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #1 (Layers and Filters)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# First group to be evaluated\n\nlayers = [1, 2, 3]\nfilter_l1 = [32, 64]\nfilter_l2 = [32, 64]\nfilter_l3 = [32, 64, 128]\n\n# Default parameters from baseline\n\n'''\nkernel_size_l1 = [5]\nkernel_size_l2 = [3]\nkernel_size_l3 = [3]\nactivation_l1 = ['relu']\nactivation_l2 = ['relu']\nactivation_l3 = ['relu']\npool_size_l1 = [2]\npool_size_l2 = [2]\npool_size_l3 = [2]\noptimizer = ['Adamax']\ndense_1 = [256]\ndense_2 = [256]\ndropout_1 = [0.25]\ndropout_2 = [0.25]\ndropout_3 = [0.4]\n'''\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #1 (Epochs: {0})...'.format(epochs))\nfor layer in layers:\n    for f_l1 in filter_l1:\n        for f_l2 in filter_l2:\n            for f_l3 in filter_l3:\n                if ((f_l3 < f_l1) or (f_l3 < f_l2) or (f_l2 < f_l1)): continue\n                digits = baseline_model(layers=layer,\n                                        filter_l1=f_l1,\n                                        filter_l2=f_l2,\n                                        filter_l3=f_l3)\n                                \n                history = digits.fit_generator(generator.flow(X_train_aux,\n                                                              y_train_aux, \n                                                              batch_size = batch_size),\n                                               epochs = epochs, \n                                               steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                               validation_data = (X_test_aux, y_test_aux),\n                                               callbacks = [lr_reduction],\n                                               verbose=0)\n                \n                print(\"Layer: {0}, Filter L1: {1:>3d}, Filter L2: {2:>3d}, Filter L3: {3:>3d} :: (Max. Val. Acc. = {4:.5f})\".format(layer, f_l1, f_l2, f_l3,\n                                                                                                                                    max(history.history['val_acc'])))\n                \n                if max(history.history['val_acc']) > max_score:\n                    best['layers'] = layer\n                    best['filter_l1'] = f_l1\n                    best['filter_l2'] = f_l2\n                    best['filter_l3'] = f_l3\n                    max_score = max(history.history['val_acc'])\n                    \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":5383},"colab_type":"code","id":"n0iJm84OUlJD","outputId":"e32fbcbc-bbdc-4616-c8d0-f0131570418b","trusted":true,"_uuid":"3f27d1f612b445c2283c87b3c134a141c99cfe7c"},"cell_type":"code","source":"print('Test #1 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: layers = {0}, filter_l1 = {1}, filter_l2 = {2}, filter_l3 = {3}'.format(best['layers'],best['filter_l1'], best['filter_l2'], best['filter_l3']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6a5c228d9173d4cb5214b3997b5e6688fac3e4c"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acda3251f80b96ffcc4ed4b3e089f5a18ca33c32"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #2 (Kernel Size & best parameters from Test #1)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# Second group to be evaluated\n\nkernel_size_l1 = [3, 4, 5]\nkernel_size_l2 = [3, 4, 5]\nkernel_size_l3 = [3, 4, 5]\n\n# Default parameters from baseline\n\n'''\nactivation_l1 = ['relu']\nactivation_l2 = ['relu']\nactivation_l3 = ['relu']\npool_size_l1 = [2]\npool_size_l2 = [2]\npool_size_l3 = [2]\noptimizer = ['Adamax']\ndense_1 = [256]\ndense_2 = [256]\ndropout_1 = [0.25]\ndropout_2 = [0.25]\ndropout_3 = [0.4]\n'''\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #2 (Epochs: {0})...'.format(epochs))\nfor k_l1 in kernel_size_l1:\n    for k_l2 in kernel_size_l2:\n        for k_l3 in kernel_size_l3:\n            digits = baseline_model(layers=best['layers'],\n                                    filter_l1=best['filter_l1'],\n                                    filter_l2=best['filter_l2'],\n                                    filter_l3=best['filter_l3'],\n                                    kernel_size_l1=k_l1,\n                                    kernel_size_l2=k_l2,\n                                    kernel_size_l3=k_l3)\n            history = digits.fit_generator(generator.flow(X_train_aux,\n                                                          y_train_aux, \n                                                          batch_size = batch_size),\n                                           epochs = epochs, \n                                           steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                           validation_data = (X_test_aux, y_test_aux),\n                                           callbacks = [lr_reduction],\n                                           verbose=0)\n                        \n            print(\"Kernel L1: {0}, Kernel L2: {1}, Kernel L3: {2} :: (Max. Val. Acc = {3:.5f})\".format(k_l1, k_l2, k_l3, max(history.history['val_acc'])))\n                       \n            if max(history.history['val_acc']) > max_score:\n                best['kernel_size_l1'] = k_l1\n                best['kernel_size_l2'] = k_l2\n                best['kernel_size_l3'] = k_l3\n                max_score = max(history.history['val_acc'])\n                            \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f442888d2baddf1f544ba99decf54e08aeb474e"},"cell_type":"code","source":"print('Test #2 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: kernel_size_l1 = {0}, kernel_size_l2 = {1}, kernel_size_l3 = {2}'.format(best['kernel_size_l1'], best['kernel_size_l2'], best['kernel_size_l3']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f279f22aaca37c66974696100f664c1e5175b6b"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf3e056d5d555a201dc4b1b114a44bdaf2d013f"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #3 (Activation functions & best parameters from Tests #1 and #2)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# Third group to be evaluated\n\nactivation_l1 = ['relu', 'sigmoid']\nactivation_l2 = ['relu', 'sigmoid']\nactivation_l3 = ['relu', 'sigmoid']\n\n# Default parameters from baseline\n\n'''\npool_size_l1 = [2]\npool_size_l2 = [2]\npool_size_l3 = [2]\noptimizer = ['Adamax']\ndense_1 = [256]\ndense_2 = [256]\ndropout_1 = [0.25]\ndropout_2 = [0.25]\ndropout_3 = [0.4]\n'''\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #3 (Epochs: {0})...'.format(epochs))\nfor act_l1 in activation_l1:\n    for act_l2 in activation_l2:\n        for act_l3 in activation_l3:\n            digits = baseline_model(layers=best['layers'],\n                                    filter_l1=best['filter_l1'],\n                                    filter_l2=best['filter_l2'],\n                                    filter_l3=best['filter_l3'],\n                                    kernel_size_l1=best['kernel_size_l1'],\n                                    kernel_size_l2=best['kernel_size_l2'],\n                                    kernel_size_l3=best['kernel_size_l3'],                                   \n                                    activation_l1=act_l1,\n                                    activation_l2=act_l2,\n                                    activation_l3=act_l3)\n            history = digits.fit_generator(generator.flow(X_train_aux,\n                                                          y_train_aux, \n                                                          batch_size = batch_size),\n                                           epochs = epochs,\n                                           steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                           validation_data = (X_test_aux, y_test_aux),\n                                           callbacks = [lr_reduction],\n                                           verbose=0)\n            \n            print(\"Act. L1: {0}, Act. L2: {1}, Act. L3: {2} :: (Max. Val. Acc = {3:.5f})\".format(act_l1, act_l2, act_l3, max(history.history['val_acc'])))\n            \n            if max(history.history['val_acc']) > max_score:\n                best['activation_l1'] = act_l1\n                best['activation_l2'] = act_l2\n                best['activation_l3'] = act_l3\n                max_score = max(history.history['val_acc'])\n                            \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffdece96162fd92242bfb5bbcd1c7bc35780db6d"},"cell_type":"code","source":"print('Test #3 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: activation_l1 = {0}, activation_l2 = {1}, activation_l3 = {2}'.format(best['activation_l1'], best['activation_l2'], best['activation_l3']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f8b6323b8eddab001cd562e34e53b7cde9464ff"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acbc44f3693351c1bff8a5faed28cd329fc9cead"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #4 (Pool Size and Dense & best parameters from Tests #1, #2 and #3)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# Forth group to be evaluated\n\npool_size_l1 = [2, 3]\npool_size_l2 = [2, 3]\npool_size_l3 = [2, 3]\ndense_1 = [128, 256]\ndense_2 = [128, 256]\n\n# Default parameters from baseline\n\n'''\noptimizer = ['Adamax']\ndropout_1 = [0.25]\ndropout_2 = [0.25]\ndropout_3 = [0.4]\n'''\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #4 (Epochs: {0})...'.format(epochs))\nfor p_l1 in pool_size_l1:\n    for p_l2 in pool_size_l2:\n        for p_l3 in pool_size_l3:\n            for d1 in dense_1:\n                for d2 in dense_2:\n                    digits = baseline_model(layers=best['layers'],\n                                            filter_l1=best['filter_l1'],\n                                            filter_l2=best['filter_l2'],\n                                            filter_l3=best['filter_l3'],\n                                            kernel_size_l1=best['kernel_size_l1'],\n                                            kernel_size_l2=best['kernel_size_l2'],\n                                            kernel_size_l3=best['kernel_size_l3'],\n                                            activation_l1=best['activation_l1'],\n                                            activation_l2=best['activation_l2'],\n                                            activation_l3=best['activation_l3'],\n                                            pool_size_l1=p_l1,\n                                            pool_size_l2=p_l2,\n                                            pool_size_l3=p_l3,\n                                            dense_1=d1,\n                                            dense_2=d2)\n\n                    history = digits.fit_generator(generator.flow(X_train_aux,\n                                                                  y_train_aux, \n                                                                  batch_size = batch_size),\n                                                   epochs = epochs, \n                                                   steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                                   validation_data = (X_test_aux, y_test_aux),\n                                                   callbacks = [lr_reduction],\n                                                   verbose=0)  \n\n                    print(\"Pool Size L1: {0}, Pool Size L2: {1}, Pool Size L3: {2}, Dense 1: {3}, Dense 2: {4} :: (Max. Val. Acc = {5:.5f})\".format(p_l1, p_l2, p_l3, d1, d2, max(history.history['val_acc'])))\n\n                    if max(history.history['val_acc']) > max_score:\n                        best['pool_size_l1'] = p_l1\n                        best['pool_size_l2'] = p_l2\n                        best['pool_size_l3'] = p_l3\n                        best['dense_1'] = d1\n                        best['dense_2'] = d2\n                        max_score = max(history.history['val_acc'])\n                \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ad2affdd9b269d2e73b16ba22562a490d247519"},"cell_type":"code","source":"print('Test #4 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: pool_size_l1 = {0}, pool_size_l2 = {1}, pool_size_l3 = {2}, dense_1 = {3}, dense_2 = {4}'.format(best['pool_size_l1'], best['pool_size_l2'], best['pool_size_l3'], best['dense_1'], best['dense_2']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0db1ae9336bd2a699a1dce8226855cb6e1fc4702"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e0de7520510b919c54ef414302e7043b22ca8da"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #5 (Dropout & best parameters from Tests #1, #2, #3 and #4)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# Fifth group to be evaluated\n\ndropout_1 = [0.25, 0.4, 0.5]\ndropout_2 = [0.25, 0.4, 0.5]\ndropout_3 = [0.25, 0.4, 0.5]\n\n# Default parameters from baseline\n\n'''\noptimizer = ['Adamax']\n'''\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #5 (Epochs: {0})...'.format(epochs))\nfor drop_1 in dropout_1:\n    for drop_2 in dropout_2:\n        for drop_3 in dropout_3:\n            digits = baseline_model(layers=best['layers'],\n                                    filter_l1=best['filter_l1'],\n                                    filter_l2=best['filter_l2'],\n                                    filter_l3=best['filter_l3'],\n                                    kernel_size_l1=best['kernel_size_l1'],\n                                    kernel_size_l2=best['kernel_size_l2'],\n                                    kernel_size_l3=best['kernel_size_l3'],\n                                    activation_l1=best['activation_l1'],\n                                    activation_l2=best['activation_l2'],\n                                    activation_l3=best['activation_l3'],\n                                    pool_size_l1=best['pool_size_l1'],\n                                    pool_size_l2=best['pool_size_l2'],\n                                    pool_size_l3=best['pool_size_l3'],\n                                    dense_1=best['dense_1'],\n                                    dense_2=best['dense_2'],\n                                    dropout_1=drop_1,\n                                    dropout_2=drop_2,\n                                    dropout_3=drop_3)\n            \n            history = digits.fit_generator(generator.flow(X_train_aux,\n                                                          y_train_aux, \n                                                          batch_size = batch_size),\n                                           epochs = epochs, \n                                           steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                           validation_data = (X_test_aux, y_test_aux),\n                                           callbacks = [lr_reduction],\n                                           verbose=0)  \n                    \n            print(\"Drop 1: {0}, Drop 2: {1}, Drop 3: {2} :: (Max. Val. Acc = {3:.5f})\".format(drop_1, drop_2, drop_3, max(history.history['val_acc'])))\n            \n            if max(history.history['val_acc']) > max_score:\n                best['dropout_1'] = drop_1\n                best['dropout_2'] = drop_2\n                best['dropout_3'] = drop_3\n                max_score = max(history.history['val_acc'])\n                        \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68e65ebbb7b22079ec66cd76150ae36a9b164778"},"cell_type":"code","source":"print('Test #5 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: dropout_1 = {0}, dropout_2 = {1}, dropout_3 = {2}'.format(best['dropout_1'], best['dropout_2'], best['dropout_3']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91934794db8de8dff7da794141e5401040399e05"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34160092124529a69f6d1e7f9d824ea658561741"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #6 (Optimizers & best parameters from Tests #1, #2, #3, #4 and #5)\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Define the grid search parameters\n\n# Sixth group to be evaluated\n\noptimizer = ['Adamax', 'Adam', 'RMSProp', 'sgd', 'Adagrad', 'Adadelta', 'Nadam']\n\n# Data Augmentation\ngenerator.fit(X_train_aux)\n\nstart_time = time()\nprint('Starting Test #6 (Epochs: {0})...'.format(epochs))\nfor opt in optimizer:\n    digits = baseline_model(layers=best['layers'],\n                            filter_l1=best['filter_l1'],\n                            filter_l2=best['filter_l2'],\n                            filter_l3=best['filter_l3'],\n                            kernel_size_l1=best['kernel_size_l1'],\n                            kernel_size_l2=best['kernel_size_l2'],\n                            kernel_size_l3=best['kernel_size_l3'],\n                            activation_l1=best['activation_l1'],\n                            activation_l2=best['activation_l2'],\n                            activation_l3=best['activation_l3'],\n                            pool_size_l1=best['pool_size_l1'],\n                            pool_size_l2=best['pool_size_l2'],\n                            pool_size_l3=best['pool_size_l3'],\n                            dense_1=best['dense_1'],\n                            dense_2=best['dense_2'],\n                            dropout_1=best['dropout_1'],\n                            dropout_2=best['dropout_2'],\n                            dropout_3=best['dropout_3'],\n                            optimizer=opt)\n    history = digits.fit_generator(generator.flow(X_train_aux,\n                                                      y_train_aux, \n                                                      batch_size = batch_size),\n                                       epochs = epochs, \n                                       steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                       validation_data = (X_test_aux, y_test_aux),\n                                       callbacks = [lr_reduction],\n                                       verbose=0)\n    print(\"Optimizer: {0} :: (Max. Val. Acc = {1:.5f})\".format(opt, max(history.history['val_acc'])))\n        \n    if max(history.history['val_acc']) > max_score:\n        best['optimizer'] = opt\n        max_score = max(history.history['val_acc'])\n                        \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"080aa784740a1d7b59b203ed66a4a2f1541d32a6"},"cell_type":"code","source":"print('Test #6 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: optimizer = {0}'.format(best['optimizer']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aebabe981e8012b96a3efbcb340f0f28ee3b2ff1"},"cell_type":"code","source":"X_train_aux, X_test_aux, y_train_aux, y_test_aux = train_test_split(X_train, y_train, test_size = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7933ad454f3bf22385d8119b6d1dec622ff2a4a8"},"cell_type":"code","source":"# -----------------------------------------------------\n# Test #7 (Image Generator - Data Augmentation\n# -----------------------------------------------------\n\nmax_score = 0.0\n\n# Seventh group to be evaluated\n\naugmentation = [5, 10, 15, 20]\n\ndigits = baseline_model(layers=best['layers'],\n                        filter_l1=best['filter_l1'],\n                        filter_l2=best['filter_l2'],\n                        filter_l3=best['filter_l3'],\n                        kernel_size_l1=best['kernel_size_l1'],\n                        kernel_size_l2=best['kernel_size_l2'],\n                        kernel_size_l3=best['kernel_size_l3'],\n                        activation_l1=best['activation_l1'],\n                        activation_l2=best['activation_l2'],\n                        activation_l3=best['activation_l3'],\n                        pool_size_l1=best['pool_size_l1'],\n                        pool_size_l2=best['pool_size_l2'],\n                        pool_size_l3=best['pool_size_l3'],\n                        dense_1=best['dense_1'],\n                        dense_2=best['dense_2'],\n                        dropout_1=best['dropout_1'],\n                        dropout_2=best['dropout_2'],\n                        dropout_3=best['dropout_3'],\n                        optimizer=best['optimizer'])\n\nstart_time = time()\nprint('Starting Test #7 (Epochs: {0})...'.format(epochs))\nfor aug in augmentation:\n    \n    # Data Augmentation\n    generator = ImageDataGenerator(featurewise_center = False,\n                               samplewise_center = False, \n                               featurewise_std_normalization = False,\n                               samplewise_std_normalization = False,\n                               zca_whitening = False,\n                               rotation_range = aug, # Rotate image in aug degrees\n                               zoom_range = aug/100, # Zoom image (aug% zoom) \n                               width_shift_range = aug/100, # Shift image horizontally (aug% of width)\n                               height_shift_range = aug/100, # Shift image vertically (aug% of height)\n                               horizontal_flip = False,\n                               vertical_flip = False)\n    generator.fit(X_train_aux)\n    \n    history = digits.fit_generator(generator.flow(X_train_aux,\n                                                  y_train_aux, \n                                                  batch_size = batch_size),\n                                   epochs = epochs, \n                                   steps_per_epoch = X_train_aux.shape[0] // batch_size, \n                                   validation_data = (X_test_aux, y_test_aux),\n                                   callbacks = [lr_reduction],\n                                   verbose=0)\n    print(\"Augmentation: {0} :: (Max. Val. Acc = {1:.5f})\".format(aug, max(history.history['val_acc'])))\n        \n    if max(history.history['val_acc']) > max_score:\n        best['augmentation'] = aug\n        max_score = max(history.history['val_acc'])\n                        \nend_time = time()\nduration = end_time - start_time\nhours, rest = divmod(duration, 3600)\nminutes, seconds = divmod(rest, 60)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcf9ffa5675e006b51b2e35bc025eed5aa4a985b"},"cell_type":"code","source":"print('Test #7 - duration: {0} h {1} min {2:.2f} sec'.format(int(hours), int(minutes), seconds))\nprint('-----------------------------------------------------------------------')\nprint('Best score: {0:.5f}'.format(max_score))\nprint('Best parameters: augmentation = {0}'.format(best['augmentation']))\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f4a952f93008073abe77c031a742b56c173203f"},"cell_type":"code","source":"print('-----------------------------------------------------------------------')\nprint('Best parameters of this CNN:', best)\nprint('-----------------------------------------------------------------------')","execution_count":null,"outputs":[]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":[],"name":"TestCNN_v1.ipynb","provenance":[],"toc_visible":true,"version":"0.3.2"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}