{"cells":[{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"**Dog Breeds Classification using CNNs**\n\nWe will use imported pretrained benchmark-CNN models to classify various dog breeds based on input images of dogs. We will use the Keras deep learning library and supported functionalities for our deep learning model."},{"metadata":{"_uuid":"4051adfb8bcea1abe63d64ed0e8dc7d397c54061"},"cell_type":"markdown","source":"**Import Dependencies**"},{"metadata":{"trusted":true,"_uuid":"769acde93899da65826be635f82e49a793c428eb"},"cell_type":"code","source":"%matplotlib inline\nimport datetime as dt\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\n\nfrom os import listdir, makedirs\nfrom os.path import join, exists, expanduser\nfrom tqdm import tqdm\n\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom sklearn.linear_model import LogisticRegression\n\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.vgg16 import preprocess_input, decode_predictions\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications import xception\nfrom keras.applications import inception_v3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"968887bfd4a00aa4d6e26aa6283a11d563c62ce5"},"cell_type":"code","source":"start = dt.datetime.now() # Project time-elapsed timer for workflow purposes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a8705cb991c2846ddcc0b5f16418ce71db57d0c"},"cell_type":"markdown","source":"**Setting up Keras Pretrained Models into Kaggle Kernels**"},{"metadata":{"trusted":true,"_uuid":"11ae0f75119e6b564f550dbaf7039f5a7188f8d0"},"cell_type":"code","source":"# We have to copy the pretrained models to the cache directory (~/.keras/models) where keras is looking for them\n\n# List display our pretrained models that we have prepared in our input data directory\n!ls ../input/keras-pretrained-models/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59e903e86bf24dc63407a78ffbe6eae3e68a1b68"},"cell_type":"code","source":"# Create keras cache directories in Kaggle Kernels to load the pretrained models into\ncache_dir = expanduser(join('~', '.keras')) # Cache directory\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models') # Models directory\nif not exists(models_dir):\n    makedirs(models_dir)\n    \n# Copy a selection of our pretrained models files onto the keras cache directory so Keras can access them\n# Selection includes all the models labeled notop; and both resnet50 models\n!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/\n!cp ../input/keras-pretrained-models/resnet50* ~/.keras/models/\n\n# List display our pretrained models that are located in the keras cache directory\n!ls ~/.keras/models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de08e3d03a0398af4753d53e6773de8735c9e9f5"},"cell_type":"code","source":"!ls ../input/dog-breed-identification","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6b11f8be64f32b1f66cebca31f4ebb70857553d4"},"cell_type":"markdown","source":"**Use a Subset of the Total Dataset for Faster Prototyping**\n<br>-Top 16 classes\n<br>-Can expand to include full 120 classes with more compute"},{"metadata":{"trusted":true,"_uuid":"09fd34de680613d6fa7e3a8b8821b977c7136eff"},"cell_type":"code","source":"INPUT_SIZE = 224\nNUM_CLASSES = 16\nSEED = 1987\ndata_dir = '../input/dog-breed-identification'\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\n\nprint(\"Num training examples files: \" + str(len(listdir(join(data_dir, 'train')))) + \" | Num training examples labels: \"\n      + str(len(labels)))\nprint(\"Num test examples files: \" + str(len(listdir(join(data_dir, 'test')))) + \" | Num test examples predictions: \" \n     + str(len(sample_submission)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a212cb2aa006c4090b4580d33f2b19bae799650"},"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\n\ngroup = labels.groupby(by='breed', as_index=False).agg({'id': pd.Series.nunique})\ngroup = group.sort_values('id',ascending=False)\nprint(group)\nlabels['rank'] = group['breed']\n##labels['rank'] = labels.groupby('breed').rank()['id'] <-- comment out this line\n##labels['rank'] = labels.groupby('breed').rank()['id']\n\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e07a8ab0b6c51a6bf24b03b5d2bdf37e8561c644"},"cell_type":"code","source":"def read_img(img_id, train_or_test, size):\n    \"\"\"\n    Read and resize image\n    # Args:\n        img_id: filepath string\n        train_or_test: string \"train\" or \"test\"\n        size: resize the original image\n    # Returns:\n        Image as a numpy array\n    \"\"\"\n    img = image.load_img(join(data_dir, train_or_test, '%s.jpg' % img_id), target_size = size)\n    img = image.img_to_array(img)\n    return img","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e59334aaddc75728f5cef98dfb2ac27c55323e15"},"cell_type":"markdown","source":"**Predictions of Example Images Using Pre-trained ResNet50**"},{"metadata":{"trusted":true,"_uuid":"4731977adc0f3bfa3a20966226cdc90315ce2498"},"cell_type":"code","source":"model = ResNet50(weights='imagenet')\nj = int(np.sqrt(NUM_CLASSES))\ni = int(np.ceil(1. * NUM_CLASSES / j))\nfig = plt.figure(1, figsize = (16,16))\ngrid = ImageGrid(fig, 111, nrows_ncols=(i,j), axes_pad=0.05)\nfor i, (img_id, breed) in enumerate(labels.loc[labels['rank'] == 1, ['id', 'breed']].values):\n    ax = grid[i]\n    img = read_img(img_id, 'train', (224,224))\n    ax.imshow(img/255.)\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    preds = model.predict(x)\n    _, imagenet_class_name, prob = decode_predictions(preds, top=1)[0][0]\n    ax.text(10, 180, 'ResNet50: %s (%.2f)' % (imagenet_class_name, prob), color='w', background='k', alpha=0.8)\n    ax.text(10, 200, 'LABEL: %s' % breed, color='k', backgroundcolor='w', alpha=0.8)\n    ax.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b72b55fd17ddc9f0ba218c6b5bf80926301bc24d"},"cell_type":"markdown","source":"**Extracting VGG16 Bottleneck Features**"},{"metadata":{"trusted":true,"_uuid":"064172ba27176e638365aba802d36e3eb4677a5a"},"cell_type":"code","source":"INPUT_SIZE = 224\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype = 'float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Training images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e11321c7dbc10b32372130d6bd27985b30041a9"},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\n\n# bf is abbr. for bottleneck_features\nvgg_bottleneck = VGG16(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_vgg_bf = vgg_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_vgg_bf = vgg_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('VGG training set bottleneck features shape: {} size: {:,}'.format(train_vgg_bf.shape, train_vgg_bf.size))\nprint('VGG valid set bottleneck features shape: {} size: {:,}'.format(valid_vgg_bf.shape, valid_vgg_bf.size))\nprint(\"VGG bottleneck features should be a 512-dimensional vector for each image example / prediction\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5ce6aa1494153a037114bf003355c31b9ca1a63"},"cell_type":"markdown","source":"**Logistic Regression on Extracted Bottleneck Features**\n<br>Note: We could have also attached a fully-connected layer onto the end of the pre-trained network.\n<br>This would have worked fine, although requiring more compute and fine-tuning vs simply taking the extracted feature-representation from a pre-trained network\n<br> and doing a logistic regression on the feature-vector."},{"metadata":{"trusted":true,"_uuid":"b5097953a788d2f9038ee6734401e41ba78d6899"},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_vgg_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_vgg_bf)\nvalid_preds = logreg.predict(valid_vgg_bf)\n\nprint('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1593352e352debf88d88cd8f38230b4423ee9a5e"},"cell_type":"markdown","source":"**Using Pre-trained Xception Model Instead of VGG**"},{"metadata":{"trusted":true,"_uuid":"b53c3a1e3df680c0919ca06dae55e904201ee89b"},"cell_type":"code","source":"INPUT_SIZE = 299\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = xception.preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint(\"Training images shape: {} size: {:,}\".format(x_train.shape, x_train.size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d3f6626942b4cb976b037ae38e2dffbee6b7cd4"},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\nxception_bottleneck = xception.Xception(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_x_bf = xception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_x_bf = xception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('Xception training bottleneck features shape: {} size: {:,}'.format(train_x_bf.shape, train_x_bf.size))\nprint('Xception validation bottleneck features shape: {} size: {:,}'.format(valid_x_bf.shape, valid_x_bf.size))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0c96de69b3d6c10853ddcd722552e071ab8a7c5"},"cell_type":"markdown","source":"**LogReg on Xception Bottleneck Features**"},{"metadata":{"trusted":true,"_uuid":"023440f03205148bed0562d621b0ca495254f760"},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_x_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_x_bf)\nvalid_preds = logreg.predict(valid_x_bf)\nprint(\"Validation Xception LogLoss {}\".format(log_loss(yv, valid_probs)))\nprint(\"Validation Xception Accuracy {}\".format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ef4fbc936d51afb73c71fad698d8de1ae9916d1"},"cell_type":"markdown","source":"**Using Pre-trained Inception Model Instead of VGG**"},{"metadata":{"trusted":true,"_uuid":"5a2f6df3980975c7feeec6fecf8e5e3a92e1af72"},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\ninception_bottleneck = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_i_bf = inception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_i_bf = inception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('InceptionV3 training bottleneck features shape: {} size: {:,}'.format(train_i_bf.shape, train_i_bf.size))\nprint('InceptionV3 validation bottleneck features shape: {} size: {:,}'.format(valid_i_bf.shape, valid_i_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d7a1df63761f5348893a1178b31ccc773003026"},"cell_type":"markdown","source":"**LogReg on Inception Bottleneck Features**"},{"metadata":{"trusted":true,"_uuid":"1dc74e0f8c6d9bd14b8c7514e894e119cdeba302"},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver = 'lbfgs', random_state=SEED)\nlogreg.fit(train_i_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_i_bf)\nvalid_preds = logreg.predict(valid_i_bf)\n\nprint('Validation Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3a6712f8fac004eced7dcd1be7357ef5dc73b84"},"cell_type":"markdown","source":"**LogReg on All [Concatenated] Bottleneck Features**\n<br>-Leveraging the feature-extracting capabilities of multiple pre-trained models"},{"metadata":{"trusted":true,"_uuid":"099adde9d09587e307770b0d2007f157819ad863"},"cell_type":"code","source":"X = np.hstack([train_x_bf, train_i_bf]) # This is a array-concat function that stacks horizontally instead of vertically\nV = np.hstack([valid_x_bf, valid_i_bf])\nprint(\"Full training bottleneck features shape: {} size: {:,}\".format(X.shape, X.size))\nprint(\"Full validation bottleneck features shape: {} size: {:,}\".format(V.shape, V.size))\n\nlogreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(X, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(V)\nvalid_preds = logreg.predict(V)\nprint(\"Validation Xception+Inception LogLoss {}\".format(log_loss(yv, valid_probs)))\nprint(\"Validation Xception+Inception Accuracy {}\".format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), \n                                                                        valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c5f834a85b3b0b96410f87a4ce2058a988e9ea48"},"cell_type":"markdown","source":"**Error Checking**"},{"metadata":{"trusted":true,"_uuid":"aacd58dbb51e39e57d224acc4cbc6258202a148f"},"cell_type":"code","source":"valid_breeds = (yv * range(NUM_CLASSES)).sum(axis=1)\nerror_idx = (valid_breeds != valid_preds)\nfor img_id, breed, pred in zip(labels.loc[valid_idx, 'id'].values[error_idx],\n                               [selected_breed_list[int(b)] for b in valid_preds[error_idx]],\n                               [selected_breed_list[int(b)] for b in valid_breeds[error_idx]]):\n    fix, ax = plt.subplots(figsize=(5,5,))\n    img = read_img(img_id, 'train', (299,299))\n    ax.imshow(img/255)\n    ax.text(10, 250, 'Prediction: %s' % pred, color='w', backgroundcolor='r', alpha=0.8)\n    ax.text(10, 270, 'LABEL: %s' % breed, color='k', backgroundcolor='g', alpha=0.8)\n    ax.axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8b33c912a40680a38450cf76782d6cfc36a0cc2"},"cell_type":"code","source":"end = dt.datetime.now()\nprint(\"Total time is {} seconds or {:0.1f} hours.\".format((end-start).seconds, (end-start).seconds / 3600))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af62fe67e8a4843ea55f825d3ed88abf4319b167"},"cell_type":"markdown","source":"**References:**\n<br>https://www.kaggle.com/gaborfodor/dog-breed-pretrained-keras-models-lb-0-3"},{"metadata":{"trusted":true,"_uuid":"89e99eddb52bf9e076b46e9817f5a82e50015614"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}