{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":5441,"databundleVersionId":38425,"sourceType":"competition"},{"sourceId":9900,"sourceType":"datasetVersion","datasetId":6209},{"sourceId":110819,"sourceType":"datasetVersion","datasetId":57539},{"sourceId":110930,"sourceType":"datasetVersion","datasetId":57578}],"dockerImageVersionId":11543,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"##  Keras: Transfer Learning with ResNet50 for image classification on Cats & Dogs dataset\n___\n\n### Hossam Farhoud","metadata":{"_uuid":"40733c39647bac5056ba0dada8409d6184fefb68"}},{"cell_type":"markdown","source":"This kernel is intended to be a tutorial on Keras around image files handling for Transfer Learning using pre-trained weights from ResNet50 convnet.\n\nThough loading all train & test images resized (224 x 224 x 3) in memory would have incurred ~4.9GB of memory, the plan was to batch source image data during the training, validation & testing pipeline. Keras ImageDataGenerator supports batch sourcing image data for all training, validation and testing. Actually, it is quite clean and easy to use Keras ImageDataGenerator except few limitations (listed at the end).\n\nKeras ImageDataGenerator expects labeled training images to be available in certain folder heirarchy, 'train' data was manually split into 10k for training & 2.5k for validation and re-arranged into the desired folder hierarchy. Even 'test' images had to rearranged due to a known issue in flow_from_directory.","metadata":{"_uuid":"0182259264d23ccfb3c27d530ed10fb4ba7a35da"}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline \n\nimport cv2\n\nimport os","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-12-22T23:32:57.245734Z","iopub.execute_input":"2024-12-22T23:32:57.245941Z","iopub.status.idle":"2024-12-22T23:32:57.416378Z","shell.execute_reply.started":"2024-12-22T23:32:57.245889Z","shell.execute_reply":"2024-12-22T23:32:57.415799Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Global Constants","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false,"jupyter":{"outputs_hidden":true}}},{"cell_type":"code","source":"# Fixed for our Cats & Dogs classes\nNUM_CLASSES = 2\n\n# Fixed for Cats & Dogs color images\nCHANNELS = 3\n\nIMAGE_RESIZE = 224\nRESNET50_POOLING_AVERAGE = 'avg'\nDENSE_LAYER_ACTIVATION = 'softmax'\nOBJECTIVE_FUNCTION = 'categorical_crossentropy'\n\n# Common accuracy metric for all outputs, but can use different metrics for different output\nLOSS_METRICS = ['accuracy']\n\n# EARLY_STOP_PATIENCE must be < NUM_EPOCHS\nNUM_EPOCHS = 10\nEARLY_STOP_PATIENCE = 3\n\n# These steps value should be proper FACTOR of no.-of-images in train & valid folders respectively\n# Training images processed in each step would be no.-of-train-images / STEPS_PER_EPOCH_TRAINING\nSTEPS_PER_EPOCH_TRAINING = 10\nSTEPS_PER_EPOCH_VALIDATION = 10\n\n# These steps value should be proper FACTOR of no.-of-images in train & valid folders respectively\n# NOTE that these BATCH* are for Keras ImageDataGenerator batching to fill epoch step input\nBATCH_SIZE_TRAINING = 100\nBATCH_SIZE_VALIDATION = 100\n\n# Using 1 to easily manage mapping between test_generator & prediction for submission preparation\nBATCH_SIZE_TESTING = 1","metadata":{"trusted":true,"_uuid":"d157dc1311928d1ada13ed2ef4a34c4ea5c0b538","execution":{"iopub.status.busy":"2024-12-22T23:32:57.417305Z","iopub.execute_input":"2024-12-22T23:32:57.417595Z","iopub.status.idle":"2024-12-22T23:32:57.425395Z","shell.execute_reply.started":"2024-12-22T23:32:57.417538Z","shell.execute_reply":"2024-12-22T23:32:57.424173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tensorflow.python.keras.applications import ResNet50\nfrom tensorflow.python.keras.models import Sequential\nfrom tensorflow.python.keras.layers import Dense\n\n### \n### Below systax is available with TensorFlow 1.11 onwards but this upgrade is not available for Kaggle kernel yet\n###\n#import tensorflow as tf\n#print(tf.__version__)\n#import tensorflow as tf\n#from tf.keras.applications import ResNet50\n#from tf.keras.models import Sequential","metadata":{"trusted":true,"_uuid":"ee0da10690e2848537102cb0c74f9fe19a50431b","execution":{"iopub.status.busy":"2024-12-22T23:32:57.429493Z","iopub.execute_input":"2024-12-22T23:32:57.429960Z","iopub.status.idle":"2024-12-22T23:32:57.434767Z","shell.execute_reply.started":"2024-12-22T23:32:57.429771Z","shell.execute_reply":"2024-12-22T23:32:57.433803Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ResNet50\n* Notice that resnet50 folder has 2 pre-trained weights files... xyz_tf_kernels.h5 & xyz_tf_kernels_NOTOP.h5\n* The xyz_tf_kernels.h5 weights is useful for pure prediction of test image and this prediction will rely completely on ResNet50 pre-trained weights, i.e., it does not expected any training from our side\n* Out intention in this kernel is Transfer Learning by using ResNet50 pre-trained weights except its TOP layer, i.e., the xyz_tf_kernels_NOTOP.h5 weights... Use this weights as initial weight for training new layer using train images","metadata":{"_uuid":"b651539464d36c7443a601e914479f0d2153f435"}},{"cell_type":"code","source":"resnet_weights_path = '../input/resnet50/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5'","metadata":{"trusted":true,"_uuid":"2fc47c3f887e6713b5a2be38c02b38a8ec80bf97","execution":{"iopub.status.busy":"2024-12-22T23:32:57.435644Z","iopub.execute_input":"2024-12-22T23:32:57.435853Z","iopub.status.idle":"2024-12-22T23:32:57.446354Z","shell.execute_reply.started":"2024-12-22T23:32:57.435809Z","shell.execute_reply":"2024-12-22T23:32:57.445659Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Define Our Transfer Learning Network Model Consisting of 2 Layers\n\nHere, we are preparing specification or blueprint of the TensorFlow DAG (directed acyclcic graph) for just the MODEL part.","metadata":{"_uuid":"4ebad39fff164624f5616a83cd8831308431da7d"}},{"cell_type":"code","source":"#Still not talking about our train/test data or any pre-processing.\n\nmodel = Sequential()\n\n# 1st layer as the lumpsum weights from resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\n# NOTE that this layer will be set below as NOT TRAINABLE, i.e., use it as is\nmodel.add(ResNet50(include_top = False, pooling = RESNET50_POOLING_AVERAGE, weights = resnet_weights_path))\n\n# 2nd layer as Dense for 2-class classification, i.e., dog or cat using SoftMax activation\nmodel.add(Dense(NUM_CLASSES, activation = DENSE_LAYER_ACTIVATION))\n\n# Say not to train first layer (ResNet) model as it is already trained\nmodel.layers[0].trainable = False","metadata":{"trusted":true,"_uuid":"0bb4a8eccd70874ef21f0809f478f993fa127fb2","execution":{"iopub.status.busy":"2024-12-22T23:32:57.447407Z","iopub.execute_input":"2024-12-22T23:32:57.447607Z","iopub.status.idle":"2024-12-22T23:33:08.485887Z","shell.execute_reply.started":"2024-12-22T23:32:57.447560Z","shell.execute_reply":"2024-12-22T23:33:08.485290Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true,"_uuid":"b617a28f0f89b272a0aa2af6cf72f2dd642ee052","execution":{"iopub.status.busy":"2024-12-22T23:33:08.487106Z","iopub.execute_input":"2024-12-22T23:33:08.487402Z","iopub.status.idle":"2024-12-22T23:33:08.497814Z","shell.execute_reply.started":"2024-12-22T23:33:08.487344Z","shell.execute_reply":"2024-12-22T23:33:08.497042Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Compile Our Transfer Learning Model","metadata":{"_uuid":"3587981a5b6b1d6ff8221738d51d78cafb2dd02c"}},{"cell_type":"code","source":"from tensorflow.python.keras import optimizers\n\nsgd = optimizers.SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)\nmodel.compile(optimizer = sgd, loss = OBJECTIVE_FUNCTION, metrics = LOSS_METRICS)","metadata":{"trusted":true,"_uuid":"670238611770f43a056332cc06efff44e20ad124","execution":{"iopub.status.busy":"2024-12-22T23:33:08.498963Z","iopub.execute_input":"2024-12-22T23:33:08.499232Z","iopub.status.idle":"2024-12-22T23:33:08.610902Z","shell.execute_reply.started":"2024-12-22T23:33:08.499173Z","shell.execute_reply":"2024-12-22T23:33:08.610377Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Prepare Keras Data Generators\n\nKeras *ImageDataGenerator(...)* generates batches of tensor image data with real-time data augmentation. The data will be looped over (in batches). It is useful with large dataset to source, pre-process (resize, color conversion, image augmentation, batch normalize) & supply resulting images in batches to downstream Keras modeling components, namely *fit_generator(...)* & *predict_generator(...)* -vs- *fit(...)* & *predict(...)* for small dataset.\n\nKaggle competition rule expects Dog & Cat to be labeled as 1 & 0. Keras >> ImageDataGenerator >> flow_from_directory takes in 'classes' list for mapping it to LABEL indices otherwise treats sub-folders enumerated classes in alphabetical order, i.e., Cat is 0 & Dog is 1.","metadata":{"_uuid":"8d003a99fe4ff58b8905c7f6b5286d97a852cb7a"}},{"cell_type":"code","source":"from keras.applications.resnet50 import preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator\n\nimage_size = IMAGE_RESIZE\n\n# preprocessing_function is applied on each image but only after re-sizing & augmentation (resize => augment => pre-process)\n# Each of the keras.application.resnet* preprocess_input MOSTLY mean BATCH NORMALIZATION (applied on each batch) stabilize the inputs to nonlinear activation functions\n# Batch Normalization helps in faster convergence\ndata_generator = ImageDataGenerator(preprocessing_function=preprocess_input)\n\n# flow_From_directory generates batches of augmented data (where augmentation can be color conversion, etc)\n# Both train & valid folders must have NUM_CLASSES sub-folders\ntrain_generator = data_generator.flow_from_directory(\n        '../input/catsdogs-trainvalid-80pc-prepd/trainvalidfull4keras/trainvalidfull4keras/train',\n        target_size=(image_size, image_size),\n        batch_size=BATCH_SIZE_TRAINING,\n        class_mode='categorical')\n\nvalidation_generator = data_generator.flow_from_directory(\n        '../input/catsdogs-trainvalid-80pc-prepd/trainvalidfull4keras/trainvalidfull4keras/valid',\n        target_size=(image_size, image_size),\n        batch_size=BATCH_SIZE_VALIDATION,\n        class_mode='categorical') ","metadata":{"trusted":true,"_uuid":"ac9ebd909fee81c8c8d9b9fccb6590944d8106eb","execution":{"iopub.status.busy":"2024-12-22T23:33:08.611982Z","iopub.execute_input":"2024-12-22T23:33:08.612160Z","iopub.status.idle":"2024-12-22T23:33:32.531106Z","shell.execute_reply.started":"2024-12-22T23:33:08.612127Z","shell.execute_reply":"2024-12-22T23:33:32.530522Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Max number of steps that these generator will have opportunity to process their source content\n# len(train_generator) should be 'no. of available train images / BATCH_SIZE_TRAINING'\n# len(valid_generator) should be 'no. of available train images / BATCH_SIZE_VALIDATION'\n(BATCH_SIZE_TRAINING, len(train_generator), BATCH_SIZE_VALIDATION, len(validation_generator))","metadata":{"trusted":true,"_uuid":"5014f91b6aeae661f0bc5db5d546089a07ca5b9d","execution":{"iopub.status.busy":"2024-12-22T23:33:32.532461Z","iopub.execute_input":"2024-12-22T23:33:32.532736Z","iopub.status.idle":"2024-12-22T23:33:32.539527Z","shell.execute_reply.started":"2024-12-22T23:33:32.532681Z","shell.execute_reply":"2024-12-22T23:33:32.538688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Train Our Model With Cats & Dogs Train (splitted) Data Set","metadata":{"_uuid":"f56e7cf9eae4c86c72468e322e7f00859a2ce9cd"}},{"cell_type":"code","source":"# Early stopping & checkpointing the best model in ../working dir & restoring that as our model for prediction\nfrom tensorflow.python.keras.callbacks import EarlyStopping, ModelCheckpoint\n\ncb_early_stopper = EarlyStopping(monitor = 'val_loss', patience = EARLY_STOP_PATIENCE)\ncb_checkpointer = ModelCheckpoint(filepath = '../working/best.hdf5', monitor = 'val_loss', save_best_only = True, mode = 'auto')\n","metadata":{"trusted":true,"_uuid":"2dfad78129d725f42110cde0270c32d7373d6d1d","execution":{"iopub.status.busy":"2024-12-22T23:33:32.540661Z","iopub.execute_input":"2024-12-22T23:33:32.540897Z","iopub.status.idle":"2024-12-22T23:33:32.553283Z","shell.execute_reply.started":"2024-12-22T23:33:32.540848Z","shell.execute_reply":"2024-12-22T23:33:32.552663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Grid Search is an ideal candidate for distributed machine learning\n# Pseudo code for hyperparameters Grid Search\n\n'''\nfrom sklearn.grid_search import ParameterGrid\nparam_grid = {'epochs': [5, 10, 15], 'steps_per_epoch' : [10, 20, 50]}\n\ngrid = ParameterGrid(param_grid)\n\n# Accumulate history of all permutations (may be for viewing trend) and keep watching for lowest val_loss as final model\nfor params in grid:\n    print(params)\n'''","metadata":{"trusted":true,"_uuid":"d108d3598210930bab431f43c6865488b01d467b","execution":{"iopub.status.busy":"2024-12-22T23:33:32.554394Z","iopub.execute_input":"2024-12-22T23:33:32.554605Z","iopub.status.idle":"2024-12-22T23:33:32.571855Z","shell.execute_reply.started":"2024-12-22T23:33:32.554559Z","shell.execute_reply":"2024-12-22T23:33:32.571079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fit_history = model.fit_generator(\n        train_generator,\n        steps_per_epoch=STEPS_PER_EPOCH_TRAINING,\n        epochs = NUM_EPOCHS,\n        validation_data=validation_generator,\n        validation_steps=STEPS_PER_EPOCH_VALIDATION,\n        callbacks=[cb_checkpointer, cb_early_stopper]\n)\nmodel.load_weights(\"../working/best.hdf5\")","metadata":{"trusted":true,"_uuid":"cf85fe3c0653aa56503b7da058ea8acf445eec6e","execution":{"iopub.status.busy":"2024-12-22T23:33:32.572992Z","iopub.execute_input":"2024-12-22T23:33:32.573261Z","iopub.status.idle":"2024-12-22T23:34:33.292731Z","shell.execute_reply.started":"2024-12-22T23:33:32.573206Z","shell.execute_reply":"2024-12-22T23:34:33.291986Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Training Metrics\n\nOne of the default callbacks that is registered when training all deep learning models is the History callback. It records training metrics (training accuracy, training loss, validation loss & validation accuracy) for each epoch. Note that training accuracy & loss during epoch steps are somewhat incomplete information and they are not recorded in history.\n\nObserve that training uses early stopping, hence metrics is available for epochs run, not for NUM_EPOCHS.","metadata":{"_uuid":"1ef4dd95f6d12f9277255576645e8bfce5270b81"}},{"cell_type":"code","source":"print(fit_history.history.keys())","metadata":{"trusted":true,"_uuid":"383d7a0aa87e1508a46fcebfcd5a80d7aafb840f","execution":{"iopub.status.busy":"2024-12-22T23:34:33.294052Z","iopub.execute_input":"2024-12-22T23:34:33.294235Z","iopub.status.idle":"2024-12-22T23:34:33.297790Z","shell.execute_reply.started":"2024-12-22T23:34:33.294201Z","shell.execute_reply":"2024-12-22T23:34:33.297108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" plt.figure(1, figsize = (15,8)) \n    \nplt.subplot(221)  \nplt.plot(fit_history.history['acc'])  \nplt.plot(fit_history.history['val_acc'])  \nplt.title('model accuracy')  \nplt.ylabel('accuracy')  \nplt.xlabel('epoch')  \nplt.legend(['train', 'valid']) \n    \nplt.subplot(222)  \nplt.plot(fit_history.history['loss'])  \nplt.plot(fit_history.history['val_loss'])  \nplt.title('model loss')  \nplt.ylabel('loss')  \nplt.xlabel('epoch')  \nplt.legend(['train', 'valid']) \n\nplt.show()","metadata":{"trusted":true,"_uuid":"00e51568f7e6022c6738c0e1a41080bc7152c257","execution":{"iopub.status.busy":"2024-12-22T23:34:33.298650Z","iopub.execute_input":"2024-12-22T23:34:33.298821Z","iopub.status.idle":"2024-12-22T23:34:33.705840Z","shell.execute_reply.started":"2024-12-22T23:34:33.298789Z","shell.execute_reply":"2024-12-22T23:34:33.704893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NOTE that flow_from_directory treats each sub-folder as a class which works fine for training data\n# Actually class_mode=None is a kind of workaround for test data which too must be kept in a subfolder\n\n# batch_size can be 1 or any factor of test dataset size to ensure that test dataset is samples just once, i.e., no data is left out\ntest_generator = data_generator.flow_from_directory(\n    directory = '../input/test-files-prepd/test4keras/test4keras',\n    target_size = (image_size, image_size),\n    batch_size = BATCH_SIZE_TESTING,\n    class_mode = None,\n    shuffle = False,\n    seed = 123\n)\n\n# Try batch size of 1+ in test_generator & check batch_index & filenames in resulting batches\n'''\nfor i in test_generator:\n    #print(test_generator.batch_index, test_generator.batch_size)\n    idx = (test_generator.batch_index - 1) * test_generator.batch_size\n    print(test_generator.filenames[idx : idx + test_generator.batch_size])\n'''","metadata":{"trusted":true,"_uuid":"3941e7c5d983e2ad6ae6d6da0a1415ea27e1db7e","execution":{"iopub.status.busy":"2024-12-22T23:34:33.707521Z","iopub.execute_input":"2024-12-22T23:34:33.708016Z","iopub.status.idle":"2024-12-22T23:34:45.251019Z","shell.execute_reply.started":"2024-12-22T23:34:33.707957Z","shell.execute_reply":"2024-12-22T23:34:45.250261Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Observe Prediction Time With Different Batch Size\n\nWith GPU, 97s for full prediction with batch_size=100 -vs- 264s with 1. But note that to avoid ImageDataGenerator iterator repeatability, we need to use 1 as batch_size.","metadata":{"trusted":true,"_uuid":"8c6e7f35552f87b522aa32397ffb71ea732ce80b"}},{"cell_type":"code","source":"# Reset before each call to predict\ntest_generator.reset()\n\npred = model.predict_generator(test_generator, steps = len(test_generator), verbose = 1)\n\npredicted_class_indices = np.argmax(pred, axis = 1)","metadata":{"trusted":true,"_uuid":"9911ff38ec4b82f5282f078aae63b2a510a9a6cc","execution":{"iopub.status.busy":"2024-12-22T23:34:45.252220Z","iopub.execute_input":"2024-12-22T23:34:45.252495Z","iopub.status.idle":"2024-12-22T23:36:49.930731Z","shell.execute_reply.started":"2024-12-22T23:34:45.252440Z","shell.execute_reply":"2024-12-22T23:36:49.930181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TEST_DIR = '../input/test-files-prepd/test4keras/test4keras/'\nf, ax = plt.subplots(5, 5, figsize = (15, 15))\n\nfor i in range(0,25):\n    imgBGR = cv2.imread(TEST_DIR + test_generator.filenames[i])\n    imgRGB = cv2.cvtColor(imgBGR, cv2.COLOR_BGR2RGB)\n    \n    # a if condition else b\n    predicted_class = \"Dog\" if predicted_class_indices[i] else \"Cat\"\n\n    ax[i//5, i%5].imshow(imgRGB)\n    ax[i//5, i%5].axis('off')\n    ax[i//5, i%5].set_title(\"Predicted:{}\".format(predicted_class))    \n\nplt.show()","metadata":{"trusted":true,"_uuid":"e54bc22bd1a2af7d704f9164b624b79b8cbd4f56","execution":{"iopub.status.busy":"2024-12-22T23:36:49.931802Z","iopub.execute_input":"2024-12-22T23:36:49.932020Z","iopub.status.idle":"2024-12-22T23:36:51.527305Z","shell.execute_reply.started":"2024-12-22T23:36:49.931969Z","shell.execute_reply":"2024-12-22T23:36:51.526414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"results_df = pd.DataFrame(\n    {\n        'id': pd.Series(test_generator.filenames), \n        'label': pd.Series(predicted_class_indices)\n    })\nresults_df['id'] = results_df.id.str.extract('(\\d+)')\nresults_df['id'] = pd.to_numeric(results_df['id'], errors = 'coerce')\nresults_df.sort_values(by='id', inplace = True)\n\nresults_df.to_csv('submission.csv', index=False)\nresults_df.head()","metadata":{"trusted":true,"_uuid":"39bac850385ba7129c97c1c0d389b36e7f2f0dfe","execution":{"iopub.status.busy":"2024-12-22T23:36:51.528369Z","iopub.execute_input":"2024-12-22T23:36:51.528598Z","iopub.status.idle":"2024-12-22T23:36:51.790020Z","shell.execute_reply.started":"2024-12-22T23:36:51.528552Z","shell.execute_reply":"2024-12-22T23:36:51.789245Z"}},"outputs":[],"execution_count":null}]}