{"cells": [{"metadata": {"_uuid": "b30a120404b8e104774a292b45f0902e89acef68", "_cell_guid": "657faca0-c88f-4138-8c62-cf9974e0c894"}, "source": ["# Transfer learning with pretrained Keras models\n", "\n", "Although Kernel resources were increased recently we still can not train useful CNNs without GPU. The original ImageNet set has quite a few different dog classes so we can reuse CNNs with pretrained ImageNet weights. Fortunately prediction is much faster (<1s/image) making it possible to run meaningful experiments with Kaggle Kernels."], "cell_type": "markdown"}, {"source": ["%matplotlib inline\n", "import numpy as np\n", "import pandas as pd\n", "import datetime as dt\n", "import matplotlib.pyplot as plt\n", "from mpl_toolkits.axes_grid1 import ImageGrid\n", "from os import listdir, makedirs\n", "from os.path import join, exists, expanduser\n", "from tqdm import tqdm\n", "from sklearn.metrics import log_loss, accuracy_score\n", "from keras.preprocessing import image\n", "from keras.applications.vgg16 import VGG16\n", "from keras.applications.resnet50 import ResNet50\n", "from keras.applications import xception\n", "from keras.applications import inception_v3\n", "from keras.applications.vgg16 import preprocess_input, decode_predictions\n", "from sklearn.linear_model import LogisticRegression"], "metadata": {"collapsed": true, "_uuid": "151b0f031d10c081017bee0831d1e276148b413b", "_cell_guid": "d4c4a3a8-93af-4cd2-a95b-32a2526ac3a2"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["start = dt.datetime.now()"], "metadata": {"collapsed": true, "_uuid": "928b27a4686daca6a69bcf74bb592c9e99393992", "_cell_guid": "c60a6481-e35a-49fd-b1b3-4c74f4adc472"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "21279ccec131b58e04cca5516041df6046693ec1", "_cell_guid": "9ed0e437-74af-4568-8d15-342e9adcdb5d"}, "source": ["# Use Keras Pretrained Models dataset\n", "\n", "Kernels can't use network connection to download pretrained keras model weights.\n", "This dataset helps you to apply your favorite pretrained model in the Kaggle Kernel environment. \n", "You can find more details [here](https://www.kaggle.com/gaborfodor/keras-pretrained-models).\n", "\n", "We have to copy the pretrained models to the cache directory (~/.keras/models) where keras is looking for them.\n"], "cell_type": "markdown"}, {"source": ["!ls ../input/keras-pretrained-models/"], "metadata": {"collapsed": true, "_uuid": "4838c6bd2565ff67ce2c47ba014035c7f70a9018", "_cell_guid": "d2bdd2f5-2395-43ef-a971-5472066a0d36"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["cache_dir = expanduser(join('~', '.keras'))\n", "if not exists(cache_dir):\n", "    makedirs(cache_dir)\n", "models_dir = join(cache_dir, 'models')\n", "if not exists(models_dir):\n", "    makedirs(models_dir)"], "metadata": {"collapsed": true, "_uuid": "ac04695a27c65eccb786d1e48fe229a3c1e288a8", "_cell_guid": "482686db-3424-4b64-86b9-119563d77940"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n", "!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/\n", "!cp ../input/keras-pretrained-models/resnet50* ~/.keras/models/"], "metadata": {"collapsed": true, "_uuid": "463b5222d5affca14b75657257f0efeb40be4ea0", "_cell_guid": "ed6bb30d-f37b-4662-a1c4-ed02974204aa"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["!ls ~/.keras/models"], "metadata": {"collapsed": true, "_uuid": "7506ec54c661024a9636fb247f4a05473aa9982d", "_cell_guid": "c357ada4-644c-403a-ba61-0a84c9510a89"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["!ls ../input/dog-breed-identification"], "metadata": {"collapsed": true, "_uuid": "db9e42caac1312e22a6d4b0d3eec804c74e17e16", "_cell_guid": "aa2e9829-2f0e-43ef-bc30-1023bca24579"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "48c9f0e65f26fa26375dd32289f9c59bd1353d1a", "_cell_guid": "7024f870-2c6f-4b86-a64e-adab46e34c1b"}, "source": ["# Use top 16 classes\n", "Using all the images would take more than the 1 hour kernel limit. Let's focus on the most frequent 16 breeds."], "cell_type": "markdown"}, {"source": ["INPUT_SIZE = 224\n", "NUM_CLASSES = 16\n", "SEED = 1987\n", "data_dir = '../input/dog-breed-identification'\n", "labels = pd.read_csv(join(data_dir, 'labels.csv'))\n", "sample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\n", "print(len(listdir(join(data_dir, 'train'))), len(labels))\n", "print(len(listdir(join(data_dir, 'test'))), len(sample_submission))"], "metadata": {"collapsed": true, "_uuid": "3f62e82d998e8b2ba3999542492e632c5083a901", "_cell_guid": "8bcb2bda-88dc-4ad8-9177-0c126401c3e1"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\n", "labels = labels[labels['breed'].isin(selected_breed_list)]\n", "labels['target'] = 1\n", "labels['rank'] = labels.groupby('breed').rank()['id']\n", "labels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\n", "np.random.seed(seed=SEED)\n", "rnd = np.random.random(len(labels))\n", "train_idx = rnd < 0.8\n", "valid_idx = rnd >= 0.8\n", "y_train = labels_pivot[selected_breed_list].values\n", "ytr = y_train[train_idx]\n", "yv = y_train[valid_idx]"], "metadata": {"collapsed": true, "_uuid": "ab322ce7a697d43a883b1725c7f71bf4486fd5ed", "_cell_guid": "f55b18df-3698-4146-9157-913227416e21"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["def read_img(img_id, train_or_test, size):\n", "    \"\"\"Read and resize image.\n", "    # Arguments\n", "        img_id: string\n", "        train_or_test: string 'train' or 'test'.\n", "        size: resize the original image.\n", "    # Returns\n", "        Image as numpy array.\n", "    \"\"\"\n", "    img = image.load_img(join(data_dir, train_or_test, '%s.jpg' % img_id), target_size=size)\n", "    img = image.img_to_array(img)\n", "    return img"], "metadata": {"collapsed": true, "_uuid": "7d26cc67909b5bd70173b5f2ed8352b210e06fb3", "_cell_guid": "c48fc864-d70f-4045-96eb-12de12c0ad41"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "3bfb773d8c0b538400978a63de6ea47d7fb5d4d5", "_cell_guid": "e7ee34ac-0ad9-49dc-8558-0a3ae120b287"}, "source": ["# ResNet50 class predictions for example images"], "cell_type": "markdown"}, {"source": ["model = ResNet50(weights='imagenet')\n", "j = int(np.sqrt(NUM_CLASSES))\n", "i = int(np.ceil(1. * NUM_CLASSES / j))\n", "fig = plt.figure(1, figsize=(16, 16))\n", "grid = ImageGrid(fig, 111, nrows_ncols=(i, j), axes_pad=0.05)\n", "for i, (img_id, breed) in enumerate(labels.loc[labels['rank'] == 1, ['id', 'breed']].values):\n", "    ax = grid[i]\n", "    img = read_img(img_id, 'train', (224, 224))\n", "    ax.imshow(img / 255.)\n", "    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n", "    preds = model.predict(x)\n", "    _, imagenet_class_name, prob = decode_predictions(preds, top=1)[0][0]\n", "    ax.text(10, 180, 'ResNet50: %s (%.2f)' % (imagenet_class_name , prob), color='w', backgroundcolor='k', alpha=0.8)\n", "    ax.text(10, 200, 'LABEL: %s' % breed, color='k', backgroundcolor='w', alpha=0.8)\n", "    ax.axis('off')\n", "plt.show()"], "metadata": {"collapsed": true, "_uuid": "e48ddfeed10002d48a318c564921c6e9e5b3c5f3", "_cell_guid": "f0a4e5c1-02af-4689-9845-f0f77322fb46"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "3774760723fa28d44b3505c2047c1a0b3f32eb05", "_cell_guid": "0a2cd723-1859-4b36-b5af-897b7b501c9e"}, "source": ["Preprocessing and prediction seems to be working. 75% accuracy on these 16 images."], "cell_type": "markdown"}, {"metadata": {"_uuid": "dd270f61ff0278c9171592f4999513b460f8f262", "_cell_guid": "5759352a-e324-468c-b0b9-b360962a823f"}, "source": ["# Extract VGG16 bottleneck features"], "cell_type": "markdown"}, {"source": ["INPUT_SIZE = 224\n", "POOLING = 'avg'\n", "x_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\n", "for i, img_id in tqdm(enumerate(labels['id'])):\n", "    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n", "    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n", "    x_train[i] = x\n", "print('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))"], "metadata": {"collapsed": true, "_uuid": "db4f7052673aa28ca7822b9030dba078a6afb878", "_cell_guid": "62de53fc-18b6-45f4-8ec3-14a3287b2015"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["Xtr = x_train[train_idx]\n", "Xv = x_train[valid_idx]\n", "print((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\n", "vgg_bottleneck = VGG16(weights='imagenet', include_top=False, pooling=POOLING)\n", "train_vgg_bf = vgg_bottleneck.predict(Xtr, batch_size=32, verbose=1)\n", "valid_vgg_bf = vgg_bottleneck.predict(Xv, batch_size=32, verbose=1)\n", "print('VGG train bottleneck features shape: {} size: {:,}'.format(train_vgg_bf.shape, train_vgg_bf.size))\n", "print('VGG valid bottleneck features shape: {} size: {:,}'.format(valid_vgg_bf.shape, valid_vgg_bf.size))"], "metadata": {"collapsed": true, "_uuid": "11bfa7db44dc81846a939f2c5259b283bbb7f9e4", "_cell_guid": "ff0176f1-60a6-4487-9df4-438f3604e4b8"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "8adbc32293a65ff184b659282e3c9a4a5eba64af", "_cell_guid": "68b9963a-e2cd-463c-8708-11d2685e0eb2"}, "source": ["# LogReg on VGG bottleneck features"], "cell_type": "markdown"}, {"source": ["logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\n", "logreg.fit(train_vgg_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\n", "valid_probs = logreg.predict_proba(valid_vgg_bf)\n", "valid_preds = logreg.predict(valid_vgg_bf)"], "metadata": {"collapsed": true, "_uuid": "6af396744a3c6d41f8256f993e60389d67e2ab52", "_cell_guid": "90569f5c-b196-4104-9089-43d8b8ddd12b"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["print('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\n", "print('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))"], "metadata": {"collapsed": true, "_uuid": "4213c70e4a252cfae11f792aad256664258bb6be", "_cell_guid": "c92cf2e3-bd0f-44f4-932a-8fec12bf95ea"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "0c150ebe0b4cc2fe05acb28d0f5cb0b69ceaf730", "_cell_guid": "dcf8bcd6-c73e-4677-a2a3-30bc1708dd96"}, "source": ["Not bad, 90% accuracy for the top 16 classes. The multiclass classification with 120 classes is more difficult so these LogLoss/Accuracy scores does not translate to LB."], "cell_type": "markdown"}, {"metadata": {"_uuid": "e8f19dc62979a04aaa396f9cd9b990751d372286", "_cell_guid": "c9e29a84-5cad-49d7-9290-5f7755cb1d43"}, "source": ["# Extract Xception bottleneck features"], "cell_type": "markdown"}, {"source": ["INPUT_SIZE = 299\n", "POOLING = 'avg'\n", "x_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\n", "for i, img_id in tqdm(enumerate(labels['id'])):\n", "    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n", "    x = xception.preprocess_input(np.expand_dims(img.copy(), axis=0))\n", "    x_train[i] = x\n", "print('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))"], "metadata": {"collapsed": true, "_uuid": "7e5f115592134a49b4997903ccc9e7497797be1e", "_cell_guid": "fe14c29b-42a0-45cb-a7a0-5f201c984ff6"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["Xtr = x_train[train_idx]\n", "Xv = x_train[valid_idx]\n", "print((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\n", "xception_bottleneck = xception.Xception(weights='imagenet', include_top=False, pooling=POOLING)\n", "train_x_bf = xception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\n", "valid_x_bf = xception_bottleneck.predict(Xv, batch_size=32, verbose=1)\n", "print('Xception train bottleneck features shape: {} size: {:,}'.format(train_x_bf.shape, train_x_bf.size))\n", "print('Xception valid bottleneck features shape: {} size: {:,}'.format(valid_x_bf.shape, valid_x_bf.size))"], "metadata": {"collapsed": true, "_uuid": "1508b3285ea437bf24aa765c76ea56042d406034", "_cell_guid": "37a9f47a-9a17-43cd-99b3-7f89192ccf34"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "a1bdd346837a8ad7fbbe1f9b5cb424a33055117d", "_cell_guid": "065777fc-0308-4526-823a-4915cd5024be"}, "source": ["# LogReg on Xception bottleneck features\n"], "cell_type": "markdown"}, {"source": ["logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\n", "logreg.fit(train_x_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\n", "valid_probs = logreg.predict_proba(valid_x_bf)\n", "valid_preds = logreg.predict(valid_x_bf)\n", "print('Validation Xception LogLoss {}'.format(log_loss(yv, valid_probs)))\n", "print('Validation Xception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))"], "metadata": {"collapsed": true, "_uuid": "db34ba100fb8a524487ed0b91134b6e65102e3fb", "_cell_guid": "dc8a93f1-1ee6-450d-90a3-8fa77f245b3b"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "17e0f22ff8cea58ba198e5556c472b7dc6ed84a0", "_cell_guid": "3fe2d11d-4b69-4bc1-b004-6fc84f56dd0e"}, "source": ["![](https://pics.me.me/such-wow-much-awesome-many-cool-bestest-thug-life-19337110.png)\n", "\n", "Much better! 98% accuracy 0.07 LogLoss."], "cell_type": "markdown"}, {"metadata": {"_uuid": "e17c08d790ef172e13900624df42d9ea9a43cc69", "_cell_guid": "8ca30754-3b63-4c2a-a7ed-a111a4e0b837"}, "source": ["# Extract Inception bottleneck features"], "cell_type": "markdown"}, {"source": ["Xtr = x_train[train_idx]\n", "Xv = x_train[valid_idx]\n", "print((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\n", "inception_bottleneck = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling=POOLING)\n", "train_i_bf = inception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\n", "valid_i_bf = inception_bottleneck.predict(Xv, batch_size=32, verbose=1)\n", "print('InceptionV3 train bottleneck features shape: {} size: {:,}'.format(train_i_bf.shape, train_i_bf.size))\n", "print('InceptionV3 valid bottleneck features shape: {} size: {:,}'.format(valid_i_bf.shape, valid_i_bf.size))"], "metadata": {"collapsed": true, "_uuid": "5244f45ca3093adc9034a4115d309349f8b7bc1e", "_cell_guid": "1c608857-0f1a-4132-b004-a1a3eda59d60"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "86550ce360964aa21f7cc70e1761132241e0cf55", "_cell_guid": "d7930ef0-2d75-409c-88c7-b150f384a93e"}, "source": ["# LogReg on Inception bottleneck features"], "cell_type": "markdown"}, {"source": ["logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\n", "logreg.fit(train_i_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\n", "valid_probs = logreg.predict_proba(valid_i_bf)\n", "valid_preds = logreg.predict(valid_i_bf)"], "metadata": {"collapsed": true, "_uuid": "e36b51ff6bba98246c6f380fecc33680deff88c5", "_cell_guid": "675864f8-7bfe-47d6-960e-d4bd686cf31a"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["print('Validation Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\n", "print('Validation Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))"], "metadata": {"collapsed": true, "_uuid": "8fe401e1ce9d41c617bd24045d4462a05f59eb88", "_cell_guid": "655a9cfe-a2f2-4a8a-90d4-dac18cf72027"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "ea2d8acf56c3a2796c5923e3a75d3ac64d22bce7", "_cell_guid": "b7b3f4f3-f134-4b3c-aaa0-9d7dd01b4b98"}, "source": ["# LogReg on all bottleneck features"], "cell_type": "markdown"}, {"source": ["X = np.hstack([train_x_bf, train_i_bf])\n", "V = np.hstack([valid_x_bf, valid_i_bf])\n", "print('Full train bottleneck features shape: {} size: {:,}'.format(X.shape, X.size))\n", "print('Full valid bottleneck features shape: {} size: {:,}'.format(V.shape, V.size))\n", "logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\n", "logreg.fit(X, (ytr * range(NUM_CLASSES)).sum(axis=1))\n", "valid_probs = logreg.predict_proba(V)\n", "valid_preds = logreg.predict(V)\n", "print('Validation Xception + Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\n", "print('Validation Xception + Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))"], "metadata": {"collapsed": true, "_uuid": "9356a162d8a39db89aab783d39c4012edd2a8ed5", "_cell_guid": "1c858bb8-9d3a-4f30-8c07-e08726d471ac"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"metadata": {"_uuid": "a8c185d2f7068a8d370aa239868d6a83b786960e", "_cell_guid": "d6d917af-8c5a-4a06-ab3c-c7fb327e954e"}, "source": ["Training this model on the full dataset would give 0.3 LogLoss on LB."], "cell_type": "markdown"}, {"metadata": {"_uuid": "646d482fb5c7b2a792abfbb6c05dd72a94f353a7", "_cell_guid": "28ac6c7c-99c0-4afa-b396-58a5524e6ebc"}, "source": ["# Check errors\n", "We still have a few misclassification errors."], "cell_type": "markdown"}, {"source": ["valid_breeds = (yv * range(NUM_CLASSES)).sum(axis=1)\n", "error_idx = (valid_breeds != valid_preds)\n", "for img_id, breed, pred in zip(labels.loc[valid_idx, 'id'].values[error_idx],\n", "                                [selected_breed_list[int(b)] for b in valid_preds[error_idx]],\n", "                                [selected_breed_list[int(b)] for b in valid_breeds[error_idx]]):\n", "    fig, ax = plt.subplots(figsize=(5,5))\n", "    img = read_img(img_id, 'train', (299, 299))\n", "    ax.imshow(img / 255.)\n", "    ax.text(10, 250, 'Prediction: %s' % pred, color='w', backgroundcolor='r', alpha=0.8)\n", "    ax.text(10, 270, 'LABEL: %s' % breed, color='k', backgroundcolor='g', alpha=0.8)\n", "    ax.axis('off')\n", "    plt.show()                                                    "], "metadata": {"collapsed": true, "_uuid": "17ae76c56f08f8602ce0456c9b14f33581da76d5", "_cell_guid": "ff6da045-b279-4d49-a581-41fe6525b797"}, "execution_count": null, "cell_type": "code", "outputs": []}, {"source": ["end = dt.datetime.now()\n", "print('Total time {} s.'.format((end - start).seconds))\n", "print('We almost used the one hour time limit.')"], "metadata": {"collapsed": true, "_uuid": "d888c16718067eabe7540f4d2573ffdda24887c9", "_cell_guid": "467d7924-aff9-4947-915c-77bc0a21ae13"}, "execution_count": null, "cell_type": "code", "outputs": []}], "nbformat_minor": 1, "metadata": {"language_info": {"mimetype": "text/x-python", "pygments_lexer": "ipython3", "name": "python", "nbconvert_exporter": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "version": "3.6.3"}, "kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}}, "nbformat": 4}