{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Convolutional Neural Network\n**Name Student 1:** Iddan Halevy\n**ID Student 1:** 204813307<br><br>\n**Name Student 2:** Itai Levi\n**ID Student 2:** 205414717","metadata":{"id":"lP6JLo1tGNBg"}},{"cell_type":"markdown","source":"### Import libraries","metadata":{"id":"gWZyYmS_UE_L"}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom keras.preprocessing.image import load_img,img_to_array, array_to_img, ImageDataGenerator\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.models import Sequential, load_model\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, RandomFlip, RandomRotation, RandomZoom, BatchNormalization\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\nimport time","metadata":{"id":"MxkJoQBkUIHC","execution":{"iopub.status.busy":"2022-02-27T16:30:31.780401Z","iopub.execute_input":"2022-02-27T16:30:31.781201Z","iopub.status.idle":"2022-02-27T16:30:37.447226Z","shell.execute_reply.started":"2022-02-27T16:30:31.781154Z","shell.execute_reply":"2022-02-27T16:30:37.446452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__\n\ndevice_name = tf.test.gpu_device_name()\nif device_name != '/device:GPU:0':\n  raise SystemError('GPU device not found')\nprint('Found GPU at: {}'.format(device_name))","metadata":{"id":"ZaTwK7ojXr2F","outputId":"0b27a96d-d11a-43e8-ab4b-87c1f01896fe","execution":{"iopub.status.busy":"2022-02-27T16:30:37.44877Z","iopub.execute_input":"2022-02-27T16:30:37.449062Z","iopub.status.idle":"2022-02-27T16:30:39.39044Z","shell.execute_reply.started":"2022-02-27T16:30:37.449029Z","shell.execute_reply":"2022-02-27T16:30:39.389582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing","metadata":{"id":"1E0Q3aoKUCRX"}},{"cell_type":"code","source":"dataset_dir = '../input/dog-breeds/dog-breed-new/'\nimg_height, img_width = 128, 128\n\ndatagen = ImageDataGenerator(validation_split=0.1, horizontal_flip=True)\ntraining_set = datagen.flow_from_directory(\n        dataset_dir + 'train_set',\n        target_size=(img_width, img_height),\n        subset='training',\n        shuffle=True)\n\nvalidation_set = datagen.flow_from_directory(\n        dataset_dir + 'train_set',\n        target_size=(img_width, img_height),\n        subset='validation',\n        shuffle=True)\n\ntest_datagen = ImageDataGenerator()\ntest_set = test_datagen.flow_from_directory(\n    dataset_dir + 'test_set',\n    target_size = (128,128)\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:30:39.392273Z","iopub.execute_input":"2022-02-27T16:30:39.39274Z","iopub.status.idle":"2022-02-27T16:30:41.488628Z","shell.execute_reply.started":"2022-02-27T16:30:39.392628Z","shell.execute_reply":"2022-02-27T16:30:41.487883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building NN","metadata":{"id":"-zfEzkRVXIwF"}},{"cell_type":"code","source":"def model_build(activation, num_of_conv_layers_per_block, num_of_blocks, initial_filters_count, pool_size, dropout, hidden_dense_count, label_count, data_augmentation = False, norm_batch = False, **kwargs):\n    model = Sequential()\n    \n    if data_augmentation: \n        model.add(RandomFlip(\"horizontal\"))\n        model.add(RandomRotation(0.1))\n        model.add(RandomZoom(0.1))\n    \n    # First conv layers\n    model.add(Conv2D(initial_filters_count, (3, 3), activation=activation, input_shape=(img_height, img_width, 3)))\n    for i in range(num_of_conv_layers_per_block - 1):\n        model.add(Conv2D(initial_filters_count, (3, 3), activation=activation))\n    \n    # Pooling\n    model.add(MaxPooling2D((2, 2)))\n    \n    for i in range(num_of_blocks - 1):\n    \n        # conv layers\n        for j in range(num_of_conv_layers_per_block):\n            model.add(Conv2D(initial_filters_count * (i + 1), (3, 3), activation=activation))\n\n        # Pooling\n        model.add(MaxPooling2D((pool_size, pool_size)))\n    \n    # Fully connected layers\n    model.add(Flatten())\n    model.add(Dropout(dropout))\n    model.add(Dense(hidden_dense_count, activation=activation))\n    \n    if (norm_batch):\n        model.add(BatchNormalization())\n    \n    model.add(Dense(label_count, activation='softmax'))\n    \n    return model","metadata":{"id":"3dtrScHxXQox","execution":{"iopub.status.busy":"2022-02-27T16:31:02.831243Z","iopub.execute_input":"2022-02-27T16:31:02.831503Z","iopub.status.idle":"2022-02-27T16:31:02.841096Z","shell.execute_reply.started":"2022-02-27T16:31:02.831476Z","shell.execute_reply":"2022-02-27T16:31:02.84031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training NN","metadata":{"id":"JT4u2S1_Y4WG"}},{"cell_type":"code","source":"def model_train(model, training_set, validation_set, optimizer, loss, batch_size, max_epochs=200, early_stopping_patience=20, metrics = ['accuracy'], exp_name=\"test.h5\", **kwargs):\n    start_time = time.time()\n    callbacks_list = [\n        ModelCheckpoint(filepath = exp_name, monitor = 'val_loss', save_best_only = True, verbose = 1),\n        EarlyStopping(monitor = 'val_loss', patience = early_stopping_patience, verbose = 1)\n    ]\n    \n    model.compile(optimizer = optimizer, loss = loss, metrics = metrics,)\n    history = model.fit(x = training_set, validation_data = validation_set, callbacks=callbacks_list, epochs = max_epochs, batch_size=batch_size, verbose=2, shuffle=True)\n    print(f'Training time:  {int(time.time() - start_time)} seconds')\n    return history","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:50:46.415866Z","iopub.execute_input":"2022-02-27T16:50:46.4164Z","iopub.status.idle":"2022-02-27T16:50:46.423677Z","shell.execute_reply.started":"2022-02-27T16:50:46.416365Z","shell.execute_reply":"2022-02-27T16:50:46.422755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualization of Loss and Accuracy","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"code","source":"def model_visualize(accuracy, val_accuracy, loss, val_loss):\n    # summarize history for accuracy\n    plt.plot(accuracy)\n    plt.plot(val_accuracy)\n    plt.title('Model Accuracy')\n    plt.ylabel('Accuracy')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Validation'], loc='upper left')\n    plt.show()\n    # summarize history for loss\n    plt.plot(loss)\n    plt.plot(val_loss)\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['train', 'validation'], loc='upper left')\n    plt.show()","metadata":{"pycharm":{"is_executing":true},"execution":{"iopub.status.busy":"2022-02-27T16:30:41.504348Z","iopub.status.idle":"2022-02-27T16:30:41.505206Z","shell.execute_reply.started":"2022-02-27T16:30:41.504959Z","shell.execute_reply":"2022-02-27T16:30:41.504986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Experiment number 1:","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"The convolutional model is comprised of 3 conv blocks, with one layer of Conv2D in each. The activation function in each layer is 'relu'.\n\nThe hyperparameters are configured below:","metadata":{}},{"cell_type":"code","source":"exp_name = \"exp1.h5\"\nhyper_params = {\n    \"exp_name\": exp_name,\n    \"activation\": 'relu',\n    \"num_of_conv_layers_per_block\": 1,\n    \"num_of_blocks\": 3,\n    \"initial_filters_count\": 32,\n    \"pool_size\": 2,\n    \"dropout\": 0.7,\n    \"hidden_dense_count\": 512,\n    \"data_augmentation\": False,\n    \"optimizer\": SGD(learning_rate= 1e-3),\n    \"loss\": 'categorical_crossentropy',\n    \"batch_size\": 128,\n    \"label_count\": training_set.num_classes,\n    \"norm_batch\": False\n}\n\n\nmodel = model_build(**hyper_params)\nhistory = model_train(model, training_set, validation_set, **hyper_params)\nmodel_visualize(history.history['accuracy'], history.history['val_accuracy'], history.history['loss'], history.history['val_loss'])\n\nmodel.load_weights(exp_name)\npredictions = model.evaluate(test_set)\n\nprint(f'Final model (test) accuracy: {predictions[1]}')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:57:46.959499Z","iopub.execute_input":"2022-02-27T16:57:46.959779Z","iopub.status.idle":"2022-02-27T16:58:09.138397Z","shell.execute_reply.started":"2022-02-27T16:57:46.959744Z","shell.execute_reply":"2022-02-27T16:58:09.135968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusions:\n\nWe saw very slow learning, after 100 epocs there was barely an improvement. \n\nThe model was training successfully on both the training and testing set, and the improvement amount was similar for both.\n\nThe final test accuracy was **2.33%**.\n\nIn the next experiment we will try a different optimizer.","metadata":{}},{"cell_type":"markdown","source":"# Experiment number 2:","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"In the second experiment we replaced the optimizer with Adam, and added another Conv2D layer per block to increase the model complexity.","metadata":{}},{"cell_type":"code","source":"exp_name = \"exp2.h5\"\nhyper_params = {\n    \"exp_name\": exp_name,\n    \"activation\": 'relu',\n    \"num_of_conv_layers_per_block\": 2,\n    \"num_of_blocks\": 3,\n    \"initial_filters_count\": 32,\n    \"pool_size\": 2,\n    \"dropout\": 0.7,\n    \"hidden_dense_count\": 512,\n    \"data_augmentation\": False,\n    \"optimizer\": Adam(learning_rate= 1e-3),\n    \"loss\": 'categorical_crossentropy',\n    \"batch_size\": 128,\n    \"label_count\": training_set.num_classes,\n    \"norm_batch\": False\n}\n\n\nmodel = model_build(**hyper_params)\nhistory = model_train(model, training_set, validation_set, **hyper_params)\nmodel_visualize(history.history['accuracy'], history.history['val_accuracy'], history.history['loss'], history.history['val_loss'])\n\nmodel.load_weights(exp_name)\npredictions = model.evaluate(test_set)\n\nprint(f'Final model (test) accuracy: {predictions[1]}')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:58:13.0441Z","iopub.execute_input":"2022-02-27T16:58:13.044362Z","iopub.status.idle":"2022-02-27T16:58:35.738007Z","shell.execute_reply.started":"2022-02-27T16:58:13.044334Z","shell.execute_reply":"2022-02-27T16:58:35.735688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusions:\n\nIn this experiment we are seeing overfitting already after arount epoch 15. \n\nAt the same time the learning curve was much better than the last experiment, so we will continue with Adam optimizer.\n\nThe final test accuracy was **4.5%**.\n\nWe would like to run more experiments with Adam optimizer, trying different hyper parameters.","metadata":{}},{"cell_type":"markdown","source":"# Experiment number 3:","metadata":{"pycharm":{"name":"#%% md\n"}}},{"cell_type":"markdown","source":"In the third experiment we added data augmentation, and reduced the learning rate, and increased the number of layer blocks.","metadata":{}},{"cell_type":"code","source":"exp_name = \"exp3.h5\"\nhyper_params = {\n    \"exp_name\": exp_name,\n    \"activation\" : \"relu\",\n    \"num_of_conv_layers_per_block\" : 2,\n    \"num_of_blocks\" : 4,\n    \"initial_filters_count\" : 32,\n    \"pool_size\" : 2,\n    \"dropout\" : 0.7,\n    \"hidden_dense_count\" : 512,\n    \"data_augmentation\" : True,\n    \"optimizer\" : Adam(learning_rate= 1e-4),\n    \"label_count\": training_set.num_classes,\n    \"loss\" : \"categorical_crossentropy\",\n    \"early_stopping_patience\": 20,\n    \"batch_size\" : 128,\n    \"norm_batch\" : False\n}\n\nmodel = model_build(**hyper_params)\nhistory = model_train(model, training_set, validation_set, **hyper_params)\nmodel_visualize(history.history['accuracy'], history.history['val_accuracy'], history.history['loss'], history.history['val_loss'])\n\nmodel.load_weights(exp_name)\npredictions = model.evaluate(test_set)\n\nprint(f'Final model (test) accuracy: {predictions[1]}')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:50:50.893982Z","iopub.execute_input":"2022-02-27T16:50:50.894713Z","iopub.status.idle":"2022-02-27T16:55:30.514226Z","shell.execute_reply.started":"2022-02-27T16:50:50.894678Z","shell.execute_reply":"2022-02-27T16:55:30.513374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusions:\n\nIn this experiment we set out to combat the overfitting we saw in the last experiment.\n\nFrom the results it seems we did much better with regards to overfitting, but still after about 60 epochs we see overfitting again.\n\nIn this experiment we got much better results, the final test accuracy was **9.16%**. ","metadata":{}},{"cell_type":"markdown","source":"# Experiment 4 (The Chosen architecture):","metadata":{}},{"cell_type":"markdown","source":"In the final experiment we increased the batch size and added a batch normalization layer.","metadata":{}},{"cell_type":"code","source":"exp_name = \"exp4.h5\"\nhyper_params = {\n    \"exp_name\": exp_name,\"activation\": 'relu',\n    \"num_of_conv_layers_per_block\": 2,\n    \"num_of_blocks\": 4,\n    \"initial_filters_count\": 32,\n    \"pool_size\": 2,\n    \"dropout\": 0.7,\n    \"hidden_dense_count\": 512,\n    \"label_count\": training_set.num_classes,\n    \"data_augmentation\": True,\n    \"optimizer\": Adam(learning_rate= 1e-4),\n    \"loss\": 'categorical_crossentropy',\n    \"batch_size\": 128 * 2,\n    \"norm_batch\": True,\n}\n\nmodel = model_build(**hyper_params)\nhistory = model_train(model, training_set, validation_set, **hyper_params)\nmodel_visualize(history.history['accuracy'], history.history['val_accuracy'], history.history['loss'], history.history['val_loss'])\n\nmodel.load_weights(exp_name)\npredictions = model.evaluate(test_set)\n\nprint(f'Final model (test) accuracy: {predictions[1]}')","metadata":{"execution":{"iopub.status.busy":"2022-02-27T16:58:40.243287Z","iopub.execute_input":"2022-02-27T16:58:40.243536Z","iopub.status.idle":"2022-02-27T17:00:59.763523Z","shell.execute_reply.started":"2022-02-27T16:58:40.24351Z","shell.execute_reply":"2022-02-27T17:00:59.761871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}