{"cells":[{"metadata":{"_uuid":"7ae97b0da5ffd6458f46beb47c208461aa4e4ef2"},"cell_type":"markdown","source":"# Keras Warm-up: Cats vs Dogs CNN with VGG16\nIn this notebook, I'm replicating the technique mentioned at [Building powerful image classification models using very little data](https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html). Using the pretrained model and build the model on-top of embeddings have been a norm in industry. Fine-tuning VGG model may be my next notebook.\n\nSteps:\n1. Image Data Preparation.\n2. VGG16 Image Embeddings Backfill.\n3. Training Multi-layer Perceptron Classifier.\n4. Submission.\n5. Appendix: PCA of VGG16 Embeddings (for inspection only)\n\nIf you find this notebook useful, please help vote it. Thanks!\n\n## 1. Library Import\nStandard library import. Checking if the execution environment contains GPU at our disposal."},{"metadata":{"_uuid":"e8be114b0179898542f64e5514e37c6e1e56752a","_cell_guid":"d7e2ec60-78d7-4877-a7a4-883cd96dcf09","trusted":false,"collapsed":true},"cell_type":"code","source":"import glob\nimport os, sys\nimport random\nfrom tqdm import tqdm\n\nimport numpy as np\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\nfrom keras.models import Sequential\nfrom keras.layers import Dropout, Flatten, Dense\nfrom keras import applications\n\nimport seaborn as sns\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nfrom keras import backend as K\nK.tensorflow_backend._get_available_gpus()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b31e5015eb932be65ee68815a274e52a7b4670ea"},"cell_type":"markdown","source":"## Experiment Setup\nCommon parameters that will be used below."},{"metadata":{"_uuid":"3b8049b3bb079d449d7be66f5b1bb784cfc94928","_cell_guid":"b550d16c-5916-4dd5-b35d-69d842636a13","collapsed":true,"trusted":false},"cell_type":"code","source":"train_data_dir = '../input/dogs-vs-cats-redux-kernels-edition/train'\ntest_data_dir = '../input/dogs-vs-cats-redux-kernels-edition/test'\n\n# Make sure you include https://www.kaggle.com/keras/vgg16/data as your data source\nvgg_model_path = '../input/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5'\n\nepochs = 20\nbatch_size = 20\nimg_width, img_height = 150, 150\n\ntraining_n_bound = 5000  # set to None to use the entire training dataset; it took about 2 hours at my Macbook Pro.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"177465300f38bfdfa9ff70deb8197c7b83a7f5af"},"cell_type":"markdown","source":"## Image Data Preparation\nThe goal here is to read images and convert them into numpy arrays. Here, Python generator is used to reduce some memory usage."},{"metadata":{"_uuid":"d02db6e1da455127972ed9f49a6bcdb79031cf1a","_cell_guid":"32731656-0415-4fa5-8430-95189ddcae22","collapsed":true,"trusted":true},"cell_type":"code","source":"def gen_image_label(directory):\n    ''' A generator that yields (label, id, jpg_filename) tuple.'''\n    for root, dirs, files in os.walk(directory):\n        for f in files:\n            _, ext = os.path.splitext(f)\n            if ext != '.jpg':\n                continue\n            basename = os.path.basename(f)\n            splits = basename.split('.')\n            if len(splits) == 3:\n                label, id_, ext = splits\n            else:\n                label = None\n                id_, ext = splits\n            fullname = os.path.join(root, f)\n            yield label, int(id_), fullname","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"747e8ccf026631b89d294b3dfd4bf0e2312a2268","_cell_guid":"b7f34f20-b1eb-42a4-8d0d-3dec029625f5","trusted":false,"collapsed":true},"cell_type":"code","source":"# Wrap training data into pandas' DataFrame.\nlst = list(gen_image_label(train_data_dir))\nrandom.shuffle(lst)\nif training_n_bound is not None:\n    lst = lst[:training_n_bound]\ntrain_df = pd.DataFrame(lst, columns=['label', 'id', 'filename'])\ntrain_df = train_df.sort_values(by=['label', 'id'])\ntrain_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"097c30f3b3194d12d06891cc8e93e0146cd717aa","_cell_guid":"f37c0296-8a54-4bfc-a16b-4f9ed2b1f247","trusted":false,"collapsed":true},"cell_type":"code","source":"train_df['label_code'] = train_df.label.map({'cat':0, 'dog':1})\ntrain_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a45de694740575c5538e61c2c84845ffaef5a6b7","_cell_guid":"3b4ba75e-aae3-4e41-a239-bd8f22cd225c","scrolled":true,"trusted":false,"collapsed":true},"cell_type":"code","source":"# Wrap testing data into pandas' DataFrame.\nlst = list(gen_image_label(test_data_dir))\ntest_df = pd.DataFrame(lst, columns=['label', 'id', 'filename'])\ntest_df = test_df.sort_values(by=['label', 'id'])\ntest_df['label_code'] = test_df.label.map({'cat':0, 'dog':1})\n\ntest_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f6e51ae904d8e9c8d924124878c883ff3f91b702","_cell_guid":"e5824587-587b-4fa3-a9e0-476d1c49d0b8","scrolled":false,"trusted":false,"collapsed":true},"cell_type":"code","source":"sns.countplot(train_df.label)\nplt.title('Number of training images per category')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"306416f4f7d78f26fe69d9f776c6d729401515c2","_cell_guid":"d78baed1-085c-48be-94ee-122f87e597a5","trusted":false,"collapsed":true},"cell_type":"code","source":"def display_images(label, n=5):\n    fig = plt.figure(figsize=(16, 8))\n    for j, fn in enumerate(train_df.loc[train_df.label == label].head(n).filename):\n        img = load_img(fn, target_size=(img_width, img_height))\n        fig.add_subplot(1, n, j + 1)\n        f = plt.imshow(img)\n        f.axes.get_xaxis().set_visible(False)\n        f.axes.get_yaxis().set_visible(False)\n    plt.title(label)\n    plt.show()\n\ndisplay_images('dog', 5)\ndisplay_images('cat', 5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ec3d1dc836bae59c3df9b6ec94827e9d6c013b8","_cell_guid":"a5b9e4e6-7aa5-4c86-93da-0bd0f29a439e","collapsed":true,"trusted":false},"cell_type":"code","source":"def gen_label_image_batch(df, batch_size, n_max_batch=10):\n    ''' A generator that yields image as np array, batch by batch.'''\n    stacked = None\n    img_arrays = []\n    label_arrays = []\n    n_batch = 0\n    for index, row in df.iterrows():\n        img_arrays.append(\n            img_to_array(\n                load_img(row['filename'], target_size=(img_width, img_height))))\n        label_arrays.append(row['label_code'])\n        if len(img_arrays) % batch_size == 0:\n            yield np.array(label_arrays), np.stack(img_arrays)\n            n_batch += 1\n            img_arrays = []\n            label_arrays = []\n            if n_max_batch is not None and n_batch == n_max_batch:\n                break\n    if img_arrays and label_arrays:\n        yield np.array(label_arrays), np.stack(img_arrays)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8805ec3b4eb5fe2b2cec293a1dca249b4bd2e327"},"cell_type":"markdown","source":"## VGG16 Image Embeddings Backfill\n\nIn this section, we load the images, and run through the VGG16 model to generate image embeddings. Note that we also serialize the output numpy arrays so that we can re-use it in the future."},{"metadata":{"_uuid":"077eb840683ecdcce54f27da01f55862bedd8123","_cell_guid":"ba74c4c8-6f81-4d0f-8d22-d5b5f92f97ed","collapsed":true,"trusted":false},"cell_type":"code","source":"datagen = ImageDataGenerator(rescale=1./255)\ndef gen_embedding_batch(df, batch_size, n_max_batch=None):\n    ''' A generator that yields the embeddings, batch by batch \n        The embedding comes from pretrained VGG16 model.\n    '''\n    batches = gen_label_image_batch(df, \n                                    batch_size=batch_size, \n                                    n_max_batch=n_max_batch)\n    model = applications.VGG16(include_top=False, \n                               weights=vgg_model_path)\n    for i, (label, imgs) in tqdm(enumerate(batches)):\n        generator = datagen.flow(\n            imgs,\n            label,\n            batch_size=batch_size,\n            shuffle=False)\n        embedding_batch = model.predict_generator(\n            generator, workers=4, verbose=0)\n        yield embedding_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"609f229305d948102ca868bd09c91640778875c9","_cell_guid":"d34cc4f5-8b5d-4909-982d-e3321e4b8daa","collapsed":true,"trusted":false},"cell_type":"code","source":"def gen_or_load_embedding(df, saved_embedding, force_gen=False):\n    if os.path.exists(saved_embedding) and not force_gen:\n        print('Loading embedding from %s...' % (saved_embedding,))\n        embedding = np.load(open(saved_embedding, 'rb'))\n    else:\n        embedding = np.stack(\n            gen_embedding_batch(df, \n                                batch_size=batch_size), \n            axis=0)\n        embedding = embedding.reshape(\n            [embedding.shape[0] * embedding.shape[1]] + list(embedding.shape[2:]))\n        np.save(open(saved_embedding, 'wb'), embedding)\n    return embedding","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d8ab39848958bfaca22ad5bc905becf946fda101","_cell_guid":"c1b06f5c-483d-46e2-b6b9-bfec4f4ca156","trusted":false,"collapsed":true},"cell_type":"code","source":"train_embeddings = gen_or_load_embedding(train_df, 'train_embeddings.npy', force_gen=True)\ntest_embeddings = gen_or_load_embedding(test_df, 'test_embeddings.npy', force_gen=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e02a068ba19911b5c7e1b4147f7e2096fb4d3e9a","_cell_guid":"57681d2d-0f3c-4f26-8269-ca7a8be35480","trusted":false,"collapsed":true},"cell_type":"code","source":"# Check embeddings' dimensions\n[train_embeddings.shape, test_embeddings.shape]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c1c0de775fcaffbdaab94d9daafdfbc0be5cb8a","_cell_guid":"75dd3fb9-244c-4d88-9991-88a185c2c0cd","collapsed":true,"trusted":false},"cell_type":"code","source":"train_indices = np.nonzero((train_df.id[:train_embeddings.shape[0]] % 4 != 0).values)[0]\nvalidate_indices = np.nonzero((train_df.id[:train_embeddings.shape[0]] % 4 == 0).values)[0]\ntrain_labels = train_df.label_code.values[train_indices]\nvalidation_labels = train_df.label_code.values[validate_indices]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3770ebc1ace809de44ca0599d0f11a5da1edab34"},"cell_type":"markdown","source":"## Training Multi-layer Perceptron Classifier\n\nThe model looks like this: VGG_embedding -> Flatten -> Dense -> Dense -> Sigmoid"},{"metadata":{"_uuid":"6dd3e5941ee1443e2c7ad8a9da0868552847e8e2","_cell_guid":"c953fd43-ae17-46d3-a4d3-41c6b9be0a1e","collapsed":true,"trusted":false},"cell_type":"code","source":"embedding_fc_model = 'embedding_fc_model.h5'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c781b5a308b23f4b3356a3a3eeec4cc384e163c6","_cell_guid":"56a3a5bf-7902-4efa-9920-2913638e2db4","collapsed":true,"trusted":false},"cell_type":"code","source":"model = Sequential()\nmodel.add(Flatten(input_shape=train_embeddings.shape[1:]))\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='rmsprop',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"265a4a879e299dd8e90e109c3e591cc88d95be65","_cell_guid":"af81d556-5eb0-4123-81a7-b13709fa9712","trusted":false,"collapsed":true},"cell_type":"code","source":"model.fit(train_embeddings[train_indices,:],\n          train_labels,\n          epochs=epochs,\n          batch_size=batch_size,\n          validation_data=(train_embeddings[validate_indices,:],\n                           validation_labels))\nmodel.save_weights(embedding_fc_model)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad9fdb1ee92a8d739ff8e37e10c3a22b4d897a22","_cell_guid":"249a29a4-6254-4f03-91c5-a66bb626c8d1","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.metrics import f1_score, accuracy_score\n\npred_validation = model.predict(train_embeddings[validate_indices,:])\n\nf1 = f1_score(validation_labels, pred_validation > 0.5)\nacc = accuracy_score(validation_labels, pred_validation > 0.5)\n(f1, acc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4529a85f56fcaa3cbf1ad8ab561178f85ae149e","_cell_guid":"2c74a8bf-7f30-4fb8-9acd-6e9f0a318b1a","trusted":false,"collapsed":true},"cell_type":"code","source":"pred_test = model.predict(test_embeddings)\npred_test.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"81f0118e7280b307e4af3091fe1ff63d5ad1cdac","_cell_guid":"68aa4665-8c23-4789-8aa1-a1596352a031"},"cell_type":"markdown","source":"### Let's see a couple of predicted results in the testing dataset."},{"metadata":{"_uuid":"69af40aa2f3d203af508b071518988e451447f7e","_cell_guid":"75893026-4103-407f-b7ee-ab8b67b48ce7","trusted":false,"collapsed":true},"cell_type":"code","source":"# Adjust n here if you want to see more results for testing dataset\nn = 10\nfor i, (index, row) in enumerate(test_df.iterrows()):\n    if i >= n:\n        break\n    fig = plt.figure(figsize=(8, 32))\n    img = load_img(row['filename'], target_size=(img_width, img_height))\n    subfig = fig.add_subplot(n, 1, i + 1)\n    pred = pred_test[i][0]\n    pred_label = 'dog' if pred > 0.5 else 'cat'\n    pred = pred if pred > 0.5 else 1-pred\n    plt.title('Looks like a {0} with probability {1}'.format(pred_label, pred))\n    f = plt.imshow(img)\n    f.axes.get_xaxis().set_visible(False)\n    f.axes.get_yaxis().set_visible(False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2230f46dbe432a6c7d6fa21578db70ecc6a076d0","_cell_guid":"aa1248c3-08f1-4ba9-bce6-ac1272c2b4b2"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"_uuid":"1d06f8660a6656cfd4c49b3d8e005eb5e2732fb2","_cell_guid":"3df5f2e2-915e-47b4-9027-af39ad397e7f","trusted":false,"collapsed":true},"cell_type":"code","source":"results = pd.DataFrame({'id': pd.Series(test_df.id.values[:pred_test.shape[0]]),\n                        'label': pd.Series(pred_test.T[0])})\nresults.to_csv('submission.csv', index=False)\nresults.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19614f3b1b37cdd284b12f9e0ed6f6f50440d2c5","_cell_guid":"570f1380-e525-4f24-a85c-dee207ae8904"},"cell_type":"markdown","source":"## Appendix: PCA of VGG16 Embeddings\n\nThis section is trying to inspect the PCA components for VGG16 embeddings."},{"metadata":{"_uuid":"7d8c04d61ca0d3b9c4a503b4512b1ed0377b775b","_cell_guid":"2b3c35a7-8647-41a7-bcb8-f0535a4aeb58","collapsed":true,"trusted":false},"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=3)\n\nrow_count = train_embeddings[train_indices,:].shape[0]\nembedding_d = int(train_embeddings[train_indices,:].size / row_count)\nvectors_train = train_embeddings[train_indices,:].reshape(row_count, embedding_d)\nX = pca.fit_transform(vectors_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"811cb3c3af6c4bfc4b9700d0fe522f78b2e7ecb2","_cell_guid":"0e76b06b-e075-4e11-842b-f8b033e88c9c","collapsed":true,"trusted":false},"cell_type":"code","source":"df = pd.DataFrame(np.concatenate((X,\n                                  train_labels[:train_embeddings[train_indices,:].shape[0]].reshape(train_embeddings[train_indices,:].shape[0],1)),\n                                 axis=1),\n                  columns=['X', 'Y', 'Z', 'label'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"090aa3d0eab71039d567b4c01c9505b95fa6094d","_cell_guid":"fe17ebf7-6921-495d-acd2-67fbf2b86667","trusted":false,"collapsed":true},"cell_type":"code","source":"g = sns.FacetGrid(df, hue=\"label\", size=7)\ng.map(plt.scatter, \"X\", \"Y\", alpha=.5)\ng.add_legend();\n\ng = sns.FacetGrid(df, hue=\"label\", size=7)\ng.map(plt.scatter, \"Y\", \"Z\", alpha=.5)\ng.add_legend();\n\ng = sns.FacetGrid(df, hue=\"label\", size=7)\ng.map(plt.scatter, \"X\", \"Z\", alpha=.5)\ng.add_legend();","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ec196ce5e15b36c516da5d8f813932331ab7e7f","_cell_guid":"c86eb923-ca13-466c-a8c0-7e3d5d7be01c","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}