{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nimport cv2\n\nimport os","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d157dc1311928d1ada13ed2ef4a34c4ea5c0b538"},"cell_type":"code","source":"# Fixed for our Cats & Dogs classes\nNUM_CLASSES = 2\n\n# Fixed for Cats & Dogs color images\nCHANNELS = 3\n\nIMAGE_RESIZE = 224\nRESNET50_POOLING_AVERAGE = 'avg'\nDENSE_LAYER_ACTIVATION = 'softmax'\nOBJECTIVE_FUNCTION = 'categorical_crossentropy'\n\n# Common accuracy metric for all outputs, but can use different metrics for different output\nLOSS_METRICS = ['accuracy']\n\n# EARLY_STOP_PATIENCE must be < NUM_EPOCHS\nNUM_EPOCHS = 10\nEARLY_STOP_PATIENCE = 3\n\n# These steps value should be proper FACTOR of no.-of-images in train & valid folders respectively\n# Training images processed in each step would be no.-of-train-images / STEPS_PER_EPOCH_TRAINING\nSTEPS_PER_EPOCH_TRAINING = 10\nSTEPS_PER_EPOCH_VALIDATION = 10\n\n# These steps value should be proper FACTOR of no.-of-images in train & valid folders respectively\n# NOTE that these BATCH* are for Keras ImageDataGenerator batching to fill epoch step input\nBATCH_SIZE_TRAINING = 100\nBATCH_SIZE_VALIDATION = 100\n\n# Using 1 to easily manage mapping between test_generator & prediction for submission preparation\nBATCH_SIZE_TESTING = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee0da10690e2848537102cb0c74f9fe19a50431b"},"cell_type":"code","source":"from tensorflow.python.keras.applications import ResNet50\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense\n\n### \n### Below systax is available with TensorFlow 1.11 onwards but this upgrade is not available for Kaggle kernel yet\n###\n#import tensorflow as tf\n#print(tf.__version__)\n#import tensorflow as tf\n#from tf.keras.applications import ResNet50\n#from tf.keras.models import Sequential","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b651539464d36c7443a601e914479f0d2153f435"},"cell_type":"markdown","source":"### ResNet50\n* Notice that resnet50 folder has 2 pre-trained weights files... xyz_tf_kernels.h5 & xyz_tf_kernels_NOTOP.h5\n* The xyz_tf_kernels.h5 weights is useful for pure prediction of test image and this prediction will rely completely on ResNet50 pre-trained weights, i.e., it does not expected any training from our side\n* Out intention in this kernel is Transfer Learning by using ResNet50 pre-trained weights except its TOP layer, i.e., the xyz_tf_kernels_NOTOP.h5 weights... Use this weights as initial weight for training new layer using train images"},{"metadata":{"trusted":true,"_uuid":"2fc47c3f887e6713b5a2be38c02b38a8ec80bf97"},"cell_type":"code","source":"resnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0bb4a8eccd70874ef21f0809f478f993fa127fb2"},"cell_type":"code","source":"\nmodel = Sequential()\n\n\nmodel.add(ResNet50(include_top = False, pooling = RESNET50_POOLING_AVERAGE, weights = resnet_weights_path))\n\nmodel.add(Dense(NUM_CLASSES, activation = DENSE_LAYER_ACTIVATION))\n\n# Say not to train first layer (ResNet) model as it is already trained\nmodel.layers[0].trainable = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b617a28f0f89b272a0aa2af6cf72f2dd642ee052"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"670238611770f43a056332cc06efff44e20ad124"},"cell_type":"code","source":"from tensorflow.python.keras import optimizers\n\nsgd = optimizers.SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)\nmodel.compile(optimizer = sgd, loss = OBJECTIVE_FUNCTION, metrics = LOSS_METRICS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac9ebd909fee81c8c8d9b9fccb6590944d8106eb"},"cell_type":"code","source":"from keras.applications.resnet50 import preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator\n\nimage_size = IMAGE_RESIZE\n\n# preprocessing_function is applied on each image but only after re-sizing & augmentation (resize => augment => pre-process)\n# Each of the keras.application.resnet* preprocess_input MOSTLY mean BATCH NORMALIZATION (applied on each batch) stabilize the inputs to nonlinear activation functions\n# Batch Normalization helps in faster convergence\ndata_generator = ImageDataGenerator(preprocessing_function=preprocess_input)\n\n# flow_From_directory generates batches of augmented data (where augmentation can be color conversion, etc)\n# Both train & valid folders must have NUM_CLASSES sub-folders\ntrain_generator = data_generator.flow_from_directory(\n        '../input/catsdogs-trainvalid-80pc-prepd/trainvalidfull4keras/trainvalidfull4keras/train',\n        target_size=(image_size, image_size),\n        batch_size=BATCH_SIZE_TRAINING,\n        class_mode='categorical')\n\nvalidation_generator = data_generator.flow_from_directory(\n        '../input/catsdogs-trainvalid-80pc-prepd/trainvalidfull4keras/trainvalidfull4keras/valid',\n        target_size=(image_size, image_size),\n        batch_size=BATCH_SIZE_VALIDATION,\n        class_mode='categorical') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5014f91b6aeae661f0bc5db5d546089a07ca5b9d"},"cell_type":"code","source":"# Max number of steps that these generator will have opportunity to process their source content\n# len(train_generator) should be 'no. of available train images / BATCH_SIZE_TRAINING'\n# len(valid_generator) should be 'no. of available train images / BATCH_SIZE_VALIDATION'\n(BATCH_SIZE_TRAINING, len(train_generator), BATCH_SIZE_VALIDATION, len(validation_generator))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f56e7cf9eae4c86c72468e322e7f00859a2ce9cd"},"cell_type":"markdown","source":"### Train the Model With Cats & Dogs Train "},{"metadata":{"trusted":true,"_uuid":"2dfad78129d725f42110cde0270c32d7373d6d1d"},"cell_type":"code","source":"# Early stopping & checkpointing the best model in ../working dir & restoring that as our model for prediction\nfrom tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n\ncb_early_stopper = EarlyStopping(monitor = 'val_loss', patience = EARLY_STOP_PATIENCE)\ncb_checkpointer = ModelCheckpoint(filepath = '../working/best.hdf5', monitor = 'val_loss', save_best_only = True, mode = 'auto')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d108d3598210930bab431f43c6865488b01d467b"},"cell_type":"code","source":"# Grid Search is an ideal candidate for distributed machine learning\n# Pseudo code for hyperparameters Grid Search\n\n'''\nfrom sklearn.grid_search import ParameterGrid\nparam_grid = {'epochs': [5, 10, 15], 'steps_per_epoch' : [10, 20, 50]}\n\ngrid = ParameterGrid(param_grid)\n\n# Accumulate history of all permutations (may be for viewing trend) and keep watching for lowest val_loss as final model\nfor params in grid:\n    print(params)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf85fe3c0653aa56503b7da058ea8acf445eec6e"},"cell_type":"code","source":"fit_history = model.fit_generator(\n        train_generator,\n        steps_per_epoch=STEPS_PER_EPOCH_TRAINING,\n        epochs = NUM_EPOCHS,\n        validation_data=validation_generator,\n        validation_steps=STEPS_PER_EPOCH_VALIDATION,\n        callbacks=[cb_checkpointer, cb_early_stopper]\n)\nmodel.load_weights(\"../working/best.hdf5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"383d7a0aa87e1508a46fcebfcd5a80d7aafb840f"},"cell_type":"code","source":"print(fit_history.history.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00e51568f7e6022c6738c0e1a41080bc7152c257"},"cell_type":"code","source":" plt.figure(1, figsize = (15,8)) \n    \nplt.subplot(221)  \nplt.plot(fit_history.history['acc'])  \nplt.plot(fit_history.history['val_acc'])  \nplt.title('model accuracy')  \nplt.ylabel('accuracy')  \nplt.xlabel('epoch')  \nplt.legend(['train', 'valid']) \n    \nplt.subplot(222)  \nplt.plot(fit_history.history['loss'])  \nplt.plot(fit_history.history['val_loss'])  \nplt.title('model loss')  \nplt.ylabel('loss')  \nplt.xlabel('epoch')  \nplt.legend(['train', 'valid']) \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3941e7c5d983e2ad6ae6d6da0a1415ea27e1db7e"},"cell_type":"code","source":"\n\ntest_generator = data_generator.flow_from_directory(\n    directory = '../input/test-files-prepd/test4keras/test4keras',\n    target_size = (image_size, image_size),\n    batch_size = BATCH_SIZE_TESTING,\n    class_mode = None,\n    shuffle = False,\n    seed = 123\n)\n\n# Try batch size of 1+ in test_generator & check batch_index & filenames in resulting batches\n'''\nfor i in test_generator:\n    #print(test_generator.batch_index, test_generator.batch_size)\n    idx = (test_generator.batch_index - 1) * test_generator.batch_size\n    print(test_generator.filenames[idx : idx + test_generator.batch_size])\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c6e7f35552f87b522aa32397ffb71ea732ce80b"},"cell_type":"markdown","source":"### Observe Prediction Time With Different Batch Size\n\nWith GPU, 97s for full prediction with batch_size=100 -vs- 264s with 1. But note that to avoid ImageDataGenerator iterator repeatability, we need to use 1 as batch_size."},{"metadata":{"trusted":true,"_uuid":"9911ff38ec4b82f5282f078aae63b2a510a9a6cc"},"cell_type":"code","source":"# Reset before each call to predict\ntest_generator.reset()\n\npred = model.predict_generator(test_generator, steps = len(test_generator), verbose = 1)\n\npredicted_class_indices = np.argmax(pred, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e54bc22bd1a2af7d704f9164b624b79b8cbd4f56"},"cell_type":"code","source":"TEST_DIR = '../input/test-files-prepd/test4keras/test4keras/'\nf, ax = plt.subplots(5, 5, figsize = (15, 15))\n\nfor i in range(0,25):\n    imgBGR = cv2.imread(TEST_DIR + test_generator.filenames[i])\n    imgRGB = cv2.cvtColor(imgBGR, cv2.COLOR_BGR2RGB)\n    \n    # a if condition else b\n    predicted_class = \"Dog\" if predicted_class_indices[i] else \"Cat\"\n\n    ax[i//5, i%5].imshow(imgRGB)\n    ax[i//5, i%5].axis('off')\n    ax[i//5, i%5].set_title(\"Predicted:{}\".format(predicted_class))    \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39bac850385ba7129c97c1c0d389b36e7f2f0dfe"},"cell_type":"code","source":"results_df = pd.DataFrame(\n    {\n        'id': pd.Series(test_generator.filenames), \n        'label': pd.Series(predicted_class_indices)\n    })\nresults_df['id'] = results_df.id.str.extract('(\\d+)')\nresults_df['id'] = pd.to_numeric(results_df['id'], errors = 'coerce')\nresults_df.sort_values(by='id', inplace = True)\n\nresults_df.to_csv('submission.csv', index=False)\nresults_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e5600300a330725b29c74c813135144a22cee56"},"cell_type":"markdown","source":"### Keras Limitations\n\n* [10/02/2018] The *validation_split* is not supported in *fit_generator*, hence its expects ImageDataGenerator for pre-splitted train & valid.\n* [10/02/2018] Model learning through *fit_generator* is not compatible for Sklearn *GridSearchCV* again *mostly* due to no support for *validation_split*."},{"metadata":{"_uuid":"60a34703bd652a314983948d93ee2dfe6d760ec3"},"cell_type":"markdown","source":"### Followup Plan\n\n1. Scale and pad and avoid aspect ratio change of original image through Keras ImageDataGenerator pre-processing insfrastructure\n2. Image augmentation\n3. Pipeline\n4. Distributed ML for Grid Search on Spark Cluster"},{"metadata":{"_uuid":"422beefc8f408582e85292cec3fe8225228f2ab5"},"cell_type":"markdown","source":"### References\n\n1. [Transfer Learning by Dan B](https://www.kaggle.com/dansbecker/transfer-learning)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}