{"cells":[{"metadata":{"_uuid":"b30a120404b8e104774a292b45f0902e89acef68","_cell_guid":"657faca0-c88f-4138-8c62-cf9974e0c894"},"cell_type":"markdown","source":"# Transfer learning with pretrained Keras models\n\nAlthough Kernel resources were increased recently we still can not train useful CNNs without GPU. The original ImageNet set has quite a few different dog classes so we can reuse CNNs with pretrained ImageNet weights. Fortunately prediction is much faster (<1s/image) making it possible to run meaningful experiments with Kaggle Kernels."},{"metadata":{"_uuid":"151b0f031d10c081017bee0831d1e276148b413b","_cell_guid":"d4c4a3a8-93af-4cd2-a95b-32a2526ac3a2","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport datetime as dt\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom os import listdir, makedirs\nfrom os.path import join, exists, expanduser\nfrom tqdm import tqdm\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications import xception\nfrom keras.applications import inception_v3\nfrom keras.applications.vgg16 import preprocess_input, decode_predictions\nfrom sklearn.linear_model import LogisticRegression","execution_count":1,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"928b27a4686daca6a69bcf74bb592c9e99393992","_cell_guid":"c60a6481-e35a-49fd-b1b3-4c74f4adc472","trusted":true},"cell_type":"code","source":"start = dt.datetime.now()","execution_count":25,"outputs":[]},{"metadata":{"_uuid":"21279ccec131b58e04cca5516041df6046693ec1","_cell_guid":"9ed0e437-74af-4568-8d15-342e9adcdb5d"},"cell_type":"markdown","source":"# Use Keras Pretrained Models dataset\n\nKernels can't use network connection to download pretrained keras model weights.\nThis dataset helps you to apply your favorite pretrained model in the Kaggle Kernel environment. \nYou can find more details [here](https://www.kaggle.com/gaborfodor/keras-pretrained-models).\n\nWe have to copy the pretrained models to the cache directory (~/.keras/models) where keras is looking for them.\n"},{"metadata":{"_uuid":"4838c6bd2565ff67ce2c47ba014035c7f70a9018","_cell_guid":"d2bdd2f5-2395-43ef-a971-5472066a0d36","trusted":true,"collapsed":true},"cell_type":"code","source":"!ls ../input/keras-pretrained-models/","execution_count":26,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"ac04695a27c65eccb786d1e48fe229a3c1e288a8","_cell_guid":"482686db-3424-4b64-86b9-119563d77940","trusted":true},"cell_type":"code","source":"cache_dir = expanduser(join('~', '.keras'))\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models')\nif not exists(models_dir):\n    makedirs(models_dir)","execution_count":27,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"463b5222d5affca14b75657257f0efeb40be4ea0","_cell_guid":"ed6bb30d-f37b-4662-a1c4-ed02974204aa","trusted":true},"cell_type":"code","source":"!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/\n!cp ../input/keras-pretrained-models/resnet50* ~/.keras/models/","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"7506ec54c661024a9636fb247f4a05473aa9982d","_cell_guid":"c357ada4-644c-403a-ba61-0a84c9510a89","trusted":true,"collapsed":true},"cell_type":"code","source":"!ls ~/.keras/models","execution_count":29,"outputs":[]},{"metadata":{"_uuid":"db9e42caac1312e22a6d4b0d3eec804c74e17e16","_cell_guid":"aa2e9829-2f0e-43ef-bc30-1023bca24579","trusted":true,"collapsed":true},"cell_type":"code","source":"!ls ../input/dog-breed-identification","execution_count":30,"outputs":[]},{"metadata":{"_uuid":"48c9f0e65f26fa26375dd32289f9c59bd1353d1a","_cell_guid":"7024f870-2c6f-4b86-a64e-adab46e34c1b"},"cell_type":"markdown","source":"# Use top 16 classes\nUsing all the images would take more than the 1 hour kernel limit. Let's focus on the most frequent 16 breeds."},{"metadata":{"_uuid":"3f62e82d998e8b2ba3999542492e632c5083a901","_cell_guid":"8bcb2bda-88dc-4ad8-9177-0c126401c3e1","trusted":true,"collapsed":true},"cell_type":"code","source":"INPUT_SIZE = 224\nNUM_CLASSES = 16\nSEED = 1987\ndata_dir = '../input/dog-breed-identification'\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\nprint(len(listdir(join(data_dir, 'train'))), len(labels))\nprint(len(listdir(join(data_dir, 'test'))), len(sample_submission))","execution_count":31,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"ab322ce7a697d43a883b1725c7f71bf4486fd5ed","_cell_guid":"f55b18df-3698-4146-9157-913227416e21","trusted":true},"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\nlabels['rank'] = labels.groupby('breed').rank()['id']\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]","execution_count":32,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"7d26cc67909b5bd70173b5f2ed8352b210e06fb3","_cell_guid":"c48fc864-d70f-4045-96eb-12de12c0ad41","trusted":true},"cell_type":"code","source":"def read_img(img_id, train_or_test, size):\n    \"\"\"Read and resize image.\n    # Arguments\n        img_id: string\n        train_or_test: string 'train' or 'test'.\n        size: resize the original image.\n    # Returns\n        Image as numpy array.\n    \"\"\"\n    img = image.load_img(join(data_dir, train_or_test, '%s.jpg' % img_id), target_size=size)\n    img = image.img_to_array(img)\n    return img","execution_count":33,"outputs":[]},{"metadata":{"_uuid":"3bfb773d8c0b538400978a63de6ea47d7fb5d4d5","_cell_guid":"e7ee34ac-0ad9-49dc-8558-0a3ae120b287"},"cell_type":"markdown","source":"# ResNet50 class predictions for example images"},{"metadata":{"trusted":true,"_uuid":"405e9446c34380bd5dd491d23cd5665405e30ffd","collapsed":true},"cell_type":"code","source":"\n\nmodel = ResNet50(weights='imagenet')\nmodel.save(\"model.bin\")\nmodel.save_weights('weights.h5');\n\n\njson = model.to_json()\nwith open(\"model.json\", \"w\" ) as json_file:\n    json_file.write(json)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3164a9d6b30e599ec701ac238d3a1486a5808ff7","collapsed":true},"cell_type":"code","source":"file = \"../input/mydog/IMG_3268.JPG\"\nim = plt.imread(file)\nplt.imshow(im)\nplt.show()\n\nmodel = ResNet50(weights='imagenet')\n\n\nsqlatch = image.load_img(file,  target_size=(224, 224))\nimg = image.img_to_array(sqlatch)\nprint(\"img\")\nprint(img)\n\nex = np.expand_dims(img.copy(), axis=0)\nprint(\"ex\")\nprint(ex)\nx = preprocess_input(ex)\nprint(\"x (preprocess)\")\nprint(x)\n\npreds = model.predict(x)\n_, imagenet_class_name, prob = decode_predictions(preds, top=1)[0][0]\n\nprint(imagenet_class_name)\nprint(prob)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e48ddfeed10002d48a318c564921c6e9e5b3c5f3","_cell_guid":"f0a4e5c1-02af-4689-9845-f0f77322fb46","trusted":true,"collapsed":true},"cell_type":"code","source":"model = ResNet50(weights='imagenet')\nj = int(np.sqrt(NUM_CLASSES))\ni = int(np.ceil(1. * NUM_CLASSES / j))\nfig = plt.figure(1, figsize=(16, 16))\ngrid = ImageGrid(fig, 111, nrows_ncols=(i, j), axes_pad=0.05)\nfor i, (img_id, breed) in enumerate(labels.loc[labels['rank'] == 1, ['id', 'breed']].values):\n    ax = grid[i]\n    img = read_img(img_id, 'train', (224, 224))\n    ax.imshow(img / 255.)\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    preds = model.predict(x)\n    _, imagenet_class_name, prob = decode_predictions(preds, top=1)[0][0]\n    ax.text(10, 180, 'ResNet50: %s (%.2f)' % (imagenet_class_name , prob), color='w', backgroundcolor='k', alpha=0.8)\n    ax.text(10, 200, 'LABEL: %s' % breed, color='k', backgroundcolor='w', alpha=0.8)\n    ax.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3774760723fa28d44b3505c2047c1a0b3f32eb05","_cell_guid":"0a2cd723-1859-4b36-b5af-897b7b501c9e"},"cell_type":"markdown","source":"Preprocessing and prediction seems to be working. 75% accuracy on these 16 images."},{"metadata":{"_uuid":"dd270f61ff0278c9171592f4999513b460f8f262","_cell_guid":"5759352a-e324-468c-b0b9-b360962a823f"},"cell_type":"markdown","source":"# Extract VGG16 bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"db4f7052673aa28ca7822b9030dba078a6afb878","_cell_guid":"62de53fc-18b6-45f4-8ec3-14a3287b2015","trusted":true},"cell_type":"code","source":"INPUT_SIZE = 224\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"11bfa7db44dc81846a939f2c5259b283bbb7f9e4","_cell_guid":"ff0176f1-60a6-4487-9df4-438f3604e4b8","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\nvgg_bottleneck = VGG16(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_vgg_bf = vgg_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_vgg_bf = vgg_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('VGG train bottleneck features shape: {} size: {:,}'.format(train_vgg_bf.shape, train_vgg_bf.size))\nprint('VGG valid bottleneck features shape: {} size: {:,}'.format(valid_vgg_bf.shape, valid_vgg_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8adbc32293a65ff184b659282e3c9a4a5eba64af","_cell_guid":"68b9963a-e2cd-463c-8708-11d2685e0eb2"},"cell_type":"markdown","source":"# LogReg on VGG bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"6af396744a3c6d41f8256f993e60389d67e2ab52","_cell_guid":"90569f5c-b196-4104-9089-43d8b8ddd12b","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_vgg_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_vgg_bf)\nvalid_preds = logreg.predict(valid_vgg_bf)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"4213c70e4a252cfae11f792aad256664258bb6be","_cell_guid":"c92cf2e3-bd0f-44f4-932a-8fec12bf95ea","trusted":true},"cell_type":"code","source":"print('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c150ebe0b4cc2fe05acb28d0f5cb0b69ceaf730","_cell_guid":"dcf8bcd6-c73e-4677-a2a3-30bc1708dd96"},"cell_type":"markdown","source":"Not bad, 90% accuracy for the top 16 classes. The multiclass classification with 120 classes is more difficult so these LogLoss/Accuracy scores does not translate to LB."},{"metadata":{"_uuid":"e8f19dc62979a04aaa396f9cd9b990751d372286","_cell_guid":"c9e29a84-5cad-49d7-9290-5f7755cb1d43"},"cell_type":"markdown","source":"# Extract Xception bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"7e5f115592134a49b4997903ccc9e7497797be1e","_cell_guid":"fe14c29b-42a0-45cb-a7a0-5f201c984ff6","trusted":true},"cell_type":"code","source":"INPUT_SIZE = 299\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = xception.preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"1508b3285ea437bf24aa765c76ea56042d406034","_cell_guid":"37a9f47a-9a17-43cd-99b3-7f89192ccf34","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\nxception_bottleneck = xception.Xception(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_x_bf = xception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_x_bf = xception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('Xception train bottleneck features shape: {} size: {:,}'.format(train_x_bf.shape, train_x_bf.size))\nprint('Xception valid bottleneck features shape: {} size: {:,}'.format(valid_x_bf.shape, valid_x_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1bdd346837a8ad7fbbe1f9b5cb424a33055117d","_cell_guid":"065777fc-0308-4526-823a-4915cd5024be"},"cell_type":"markdown","source":"# LogReg on Xception bottleneck features\n"},{"metadata":{"collapsed":true,"_uuid":"db34ba100fb8a524487ed0b91134b6e65102e3fb","_cell_guid":"dc8a93f1-1ee6-450d-90a3-8fa77f245b3b","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_x_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_x_bf)\nvalid_preds = logreg.predict(valid_x_bf)\nprint('Validation Xception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Xception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17e0f22ff8cea58ba198e5556c472b7dc6ed84a0","_cell_guid":"3fe2d11d-4b69-4bc1-b004-6fc84f56dd0e"},"cell_type":"markdown","source":"![](https://pics.me.me/such-wow-much-awesome-many-cool-bestest-thug-life-19337110.png)\n\nMuch better! 98% accuracy 0.07 LogLoss."},{"metadata":{"_uuid":"e17c08d790ef172e13900624df42d9ea9a43cc69","_cell_guid":"8ca30754-3b63-4c2a-a7ed-a111a4e0b837"},"cell_type":"markdown","source":"# Extract Inception bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"5244f45ca3093adc9034a4115d309349f8b7bc1e","_cell_guid":"1c608857-0f1a-4132-b004-a1a3eda59d60","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\ninception_bottleneck = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_i_bf = inception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_i_bf = inception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('InceptionV3 train bottleneck features shape: {} size: {:,}'.format(train_i_bf.shape, train_i_bf.size))\nprint('InceptionV3 valid bottleneck features shape: {} size: {:,}'.format(valid_i_bf.shape, valid_i_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86550ce360964aa21f7cc70e1761132241e0cf55","_cell_guid":"d7930ef0-2d75-409c-88c7-b150f384a93e"},"cell_type":"markdown","source":"# LogReg on Inception bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"e36b51ff6bba98246c6f380fecc33680deff88c5","_cell_guid":"675864f8-7bfe-47d6-960e-d4bd686cf31a","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_i_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_i_bf)\nvalid_preds = logreg.predict(valid_i_bf)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"8fe401e1ce9d41c617bd24045d4462a05f59eb88","_cell_guid":"655a9cfe-a2f2-4a8a-90d4-dac18cf72027","trusted":true},"cell_type":"code","source":"print('Validation Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea2d8acf56c3a2796c5923e3a75d3ac64d22bce7","_cell_guid":"b7b3f4f3-f134-4b3c-aaa0-9d7dd01b4b98"},"cell_type":"markdown","source":"# LogReg on all bottleneck features"},{"metadata":{"collapsed":true,"_uuid":"9356a162d8a39db89aab783d39c4012edd2a8ed5","_cell_guid":"1c858bb8-9d3a-4f30-8c07-e08726d471ac","trusted":true},"cell_type":"code","source":"X = np.hstack([train_x_bf, train_i_bf])\nV = np.hstack([valid_x_bf, valid_i_bf])\nprint('Full train bottleneck features shape: {} size: {:,}'.format(X.shape, X.size))\nprint('Full valid bottleneck features shape: {} size: {:,}'.format(V.shape, V.size))\nlogreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(X, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(V)\nvalid_preds = logreg.predict(V)\nprint('Validation Xception + Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Xception + Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8c185d2f7068a8d370aa239868d6a83b786960e","_cell_guid":"d6d917af-8c5a-4a06-ab3c-c7fb327e954e"},"cell_type":"markdown","source":"Training this model on the full dataset would give 0.3 LogLoss on LB."},{"metadata":{"_uuid":"646d482fb5c7b2a792abfbb6c05dd72a94f353a7","_cell_guid":"28ac6c7c-99c0-4afa-b396-58a5524e6ebc"},"cell_type":"markdown","source":"# Check errors\nWe still have a few misclassification errors."},{"metadata":{"collapsed":true,"_uuid":"17ae76c56f08f8602ce0456c9b14f33581da76d5","_cell_guid":"ff6da045-b279-4d49-a581-41fe6525b797","trusted":true},"cell_type":"code","source":"valid_breeds = (yv * range(NUM_CLASSES)).sum(axis=1)\nerror_idx = (valid_breeds != valid_preds)\nfor img_id, breed, pred in zip(labels.loc[valid_idx, 'id'].values[error_idx],\n                                [selected_breed_list[int(b)] for b in valid_preds[error_idx]],\n                                [selected_breed_list[int(b)] for b in valid_breeds[error_idx]]):\n    fig, ax = plt.subplots(figsize=(5,5))\n    img = read_img(img_id, 'train', (299, 299))\n    ax.imshow(img / 255.)\n    ax.text(10, 250, 'Prediction: %s' % pred, color='w', backgroundcolor='r', alpha=0.8)\n    ax.text(10, 270, 'LABEL: %s' % breed, color='k', backgroundcolor='g', alpha=0.8)\n    ax.axis('off')\n    plt.show()                                                    ","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"_uuid":"d888c16718067eabe7540f4d2573ffdda24887c9","_cell_guid":"467d7924-aff9-4947-915c-77bc0a21ae13","trusted":true},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Total time {} s.'.format((end - start).seconds))\nprint('We almost used the one hour time limit.')","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}