{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is my third experiment with this data set of Cats vs dogs. last two experiments are found here:\n1. https://www.kaggle.com/yasermarey/finding-least-complex-cnn-with-best-performance\n2. https://www.kaggle.com/yasermarey/transfer-learning-inception-v3-on-cats-vs-dogs\nThis time I would like to implement K-Folds Cross Validation. Cross Validation helps to reduce biase and therefore stablize performance between Training/Validation and Testing. I am using the CNN architecture I concluded from my last Notebook number 1 in the list above.","metadata":{}},{"cell_type":"markdown","source":"As I did in other two notebooks, here I am extracting the train.zip under tmp folder I created.I am creating two folders traing and test. Training images will used for K-Folds Cross Validation, while I am using Test images to evaluate the model at the end.           ","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0"}},{"cell_type":"markdown","source":"#### First I am importing basic libraries","metadata":{}},{"cell_type":"code","source":"import os\nfrom os import makedirs\nimport zipfile\nimport numpy as np \nimport pandas as pd \nfrom shutil import rmtree\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Extracting Data Zip File","metadata":{}},{"cell_type":"code","source":"\ndef make_directory(dir_path):\n    if os.path.exists(dir_path):\n        rmtree(dir_path)\n    makedirs(dir_path)\n    print(dir_path, ' folder is created')\n    \ninput_zip_dir = '../input/dogs-vs-cats'\nbase_dir = '../output/cats-vs-dogs'\ntmp_dir = '../output/tmp'\n\nmake_directory(base_dir)\nmake_directory(tmp_dir)\n\n# extract train data\nzip_ref = zipfile.ZipFile(os.path.join(input_zip_dir,'train.zip'), 'r')\nzip_ref.extractall(tmp_dir)\nzip_ref.close()\nprint('Done')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Creating Folders","metadata":{}},{"cell_type":"code","source":"print('Creating folders ....')\ntrain_dir = os.path.join(base_dir, 'train')\ntest_dir = os.path.join(base_dir, 'test')\nmake_directory(train_dir)\nmake_directory(test_dir)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_of_fnames = os.listdir(os.path.join(tmp_dir,'train'))\nprint('Total number of of images in tmp/train is {0}'.format(len(list_of_fnames)))\nlist_of_cats_fnames = [i for i in list_of_fnames if 'CAT' in i.upper()]\nlist_of_dogs_fnames = [i for i in list_of_fnames if 'DOG' in i.upper()]\nTOTAL_CATS = len(list_of_cats_fnames)\nTOTAL_DOGS = len(list_of_dogs_fnames)\nprint('{0} CATS images'.format(TOTAL_CATS))\nprint('{0} DOGS images'.format(TOTAL_DOGS))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Declaring Constants","metadata":{}},{"cell_type":"code","source":"TRAIN_TEST_SPLIT_AT = 0.9\nBATCH_SIZE = 100\nTARGET_SIZE = (128, 128)\nNO_OF_EPOCHS = 1\nEXPERIMENT_SIZE = 10000\nNO_OF_FOLDS = 5","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\nDistributing images to \\n {0} \\n {1} \\n'\n      '\\nsuch that {2}% of total number of images goes to training and \\n'\n      '{3}% goes to test'.format(\n    train_dir,test_dir,\n    round(TRAIN_TEST_SPLIT_AT * 100),\n    round((1 - TRAIN_TEST_SPLIT_AT) * 100)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Copying Image files[](http://)","metadata":{}},{"cell_type":"code","source":"# Copy images from tmp_dir to train/Cats, train/Dogs and to validation/Cats and validation/Dogs\n# according to the split percentage we decided.\n\nfrom shutil import copyfile\n\nnp.random.shuffle(list_of_cats_fnames)\nnp.random.shuffle(list_of_dogs_fnames)\n\ntmp_train_dir = os.path.join(tmp_dir, 'train')\nc = 0\nfor i in list_of_cats_fnames:\n    if c < (round(TRAIN_TEST_SPLIT_AT * EXPERIMENT_SIZE)):\n        copyfile(os.path.join(tmp_train_dir, i), os.path.join(train_dir, i))\n    else:\n        copyfile(os.path.join(tmp_train_dir, i), os.path.join(test_dir, i))\n    c += 1\n    if c >= EXPERIMENT_SIZE:\n        break\n\nc = 0\nfor i in list_of_dogs_fnames:\n    if c < (round(TRAIN_TEST_SPLIT_AT * EXPERIMENT_SIZE)):\n        copyfile(os.path.join(tmp_train_dir, i), os.path.join(train_dir, i))\n    else:\n        copyfile(os.path.join(tmp_train_dir, i), os.path.join(test_dir, i))\n    c += 1\n    if c >= EXPERIMENT_SIZE:\n        break\n\nprint('Total training cat images :', len(os.listdir(train_dir)))\nprint('Total test dog images :', len(os.listdir(test_dir)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prepare Images","metadata":{}},{"cell_type":"code","source":"# Creat train_X for a numpy array for training images file names and create correpsonding train_labels\ntrain_X = [img_fname for img_fname in os.listdir(train_dir)]\ntrain_X = np.array(train_X)\n# \ntrain_labels = [l.split('/')[-1].split('.')[0].strip('0123456789') for l in train_X]\ntrain_labels = np.array(train_labels)\n# \nprint ('Training shape:', train_X.shape, train_labels.shape) \n# \nprint(train_X[:5], train_labels[:5])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Construct CNN Architecture","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.optimizers import RMSprop\nprint('Constructing and compiling model ...')\nmodel = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(8, (3, 3), activation='relu', input_shape=(128, 128, 3)),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Conv2D(16, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(1, activation='sigmoid')\n])\nmodel.compile(optimizer=RMSprop(lr=0.001), loss='binary_crossentropy', metrics=['accuracy'])\nprint('Done')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Train and Cross Validate","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nimport sklearn.model_selection as sklrn\n\ndef train_and_cross_validate (model, x_data, y_data, n_folds=NO_OF_FOLDS, epochs=NO_OF_EPOCHS, batch_size=BATCH_SIZE):\n    # \n    scores = []\n    \n    #  Loading images through generators ...\n    train_datagen = ImageDataGenerator(rescale=1. / 255.,\n                                       rotation_range=40,\n                                       width_shift_range=0.2,\n                                       height_shift_range=0.2,\n                                       shear_range=0.2,\n                                       zoom_range=0.2,\n                                       horizontal_flip=True)    \n    validation_datagen = ImageDataGenerator(rescale=1. / 255)\n          \n    # prepare cross validation\n    kfold = sklrn.KFold(n_folds, shuffle=True, random_state=1)\n    # enumerate splits\n    FoldsSetNo = 0 \n    for train_ix, test_ix in kfold.split(x_data):\n        print ('Folds Set # {0}'.format(FoldsSetNo))\n        # select rows for train and test\n        xx_train, yy_train, xx_test, yy_test = \\\n            x_data[train_ix], y_data[train_ix], x_data[test_ix], y_data[test_ix]\n\n        # flow training images in batches for the current folds set\n        # for training         \n        train_generator = train_datagen.flow_from_dataframe(\n            dataframe = pd.DataFrame({'id':xx_train,'label':yy_train}), \n            directory=train_dir, \n            x_col='id',\n            y_col='label',\n            batch_size=batch_size,\n            target_size=TARGET_SIZE,\n            class_mode='binary',\n            shuffle = False)\n        \n        # and for validation         \n        validation_generator = validation_datagen.flow_from_dataframe(\n            dataframe = pd.DataFrame({'id':xx_test,'label':yy_test}), \n            directory=train_dir, \n            x_col='id',\n            y_col='label',\n            batch_size=batch_size,\n            target_size=TARGET_SIZE,\n            class_mode='binary',\n            shuffle=False)\n\n        # fit the model\n        history = model.fit(train_generator,\n                            epochs=epochs,  # The more we train the more our model fits the data\n                            batch_size=batch_size,  # Smaller batch sizes = samller steps towards convergence\n                            validation_data=validation_generator,\n                            verbose=1)\n        # store scores\n        scores.append({'acc':np.average(history.history['accuracy']),'val_acc':np.average(history.history['val_accuracy'])})\n        FoldsSetNo +=1\n    return scores\nprint('Starting training and k-fold cross validation ...')\nscores = train_and_cross_validate(model, train_X, train_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from matplotlib import pyplot as plt\n# summarize history for accuracy\n# print(scores)\ntrain = []\nvalidation = []\nplt.subplot(1, 1, 1)\nfor s in scores:\n    train.append(s['acc'])\n    validation.append(s['val_acc'])\nprint(train)\nprint(validation)\nplt.plot(train, color='blue', label='train')\nplt.plot(validation , color='red', label='validation')\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper left')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Evaluate the Model","metadata":{"trusted":true}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}