{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Import Dependencies**","metadata":{"_uuid":"4051adfb8bcea1abe63d64ed0e8dc7d397c54061"}},{"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\n\nfrom os import listdir, makedirs\nfrom os.path import join, exists, expanduser\nfrom tqdm import tqdm # Fancy progress bars\n\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom sklearn.linear_model import LogisticRegression\n\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.vgg16 import preprocess_input, decode_predictions\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications import xception\nfrom keras.applications import inception_v3","metadata":{"_uuid":"769acde93899da65826be635f82e49a793c428eb","execution":{"iopub.status.busy":"2023-10-17T14:33:30.951942Z","iopub.execute_input":"2023-10-17T14:33:30.952354Z","iopub.status.idle":"2023-10-17T14:33:33.649918Z","shell.execute_reply.started":"2023-10-17T14:33:30.952277Z","shell.execute_reply":"2023-10-17T14:33:33.648300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loading up Keras Pretrained Models into Kaggle Kernels**","metadata":{"_uuid":"9a8705cb991c2846ddcc0b5f16418ce71db57d0c"}},{"cell_type":"code","source":"# This project was done in Kaggle Kernels, where we'd need to copy the Keras pretrained models\n# into the cache directory (~/.keras/models) where keras is looking for them\n\n# Display the pretrained models that we have prepared in our file directory\n!ls ../input/keras-pretrained-models/","metadata":{"_uuid":"11ae0f75119e6b564f550dbaf7039f5a7188f8d0","execution":{"iopub.status.busy":"2023-10-17T14:33:33.651705Z","iopub.execute_input":"2023-10-17T14:33:33.652057Z","iopub.status.idle":"2023-10-17T14:33:34.834619Z","shell.execute_reply.started":"2023-10-17T14:33:33.651985Z","shell.execute_reply":"2023-10-17T14:33:34.833229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create keras cache directories in Kaggle Kernels to load the pretrained models into\ncache_dir = expanduser(join('~', '.keras')) # Cache directory\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models') # Models directory\nif not exists(models_dir):\n    makedirs(models_dir)\n    \n# Copy a selection of our pretrained models files onto the keras cache directory so Keras can access them\n# Selection includes all the models labeled notop; and both resnet50 models\n!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/\n!cp ../input/keras-pretrained-models/resnet50* ~/.keras/models/\n\n# Display our pretrained models that are located in the keras cache directory\n!ls ~/.keras/models","metadata":{"_uuid":"59e903e86bf24dc63407a78ffbe6eae3e68a1b68","execution":{"iopub.status.busy":"2023-10-17T14:33:34.836945Z","iopub.execute_input":"2023-10-17T14:33:34.837433Z","iopub.status.idle":"2023-10-17T14:33:48.930860Z","shell.execute_reply.started":"2023-10-17T14:33:34.837350Z","shell.execute_reply":"2023-10-17T14:33:48.929510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls ../input/dog-breed-identification","metadata":{"_uuid":"de08e3d03a0398af4753d53e6773de8735c9e9f5","execution":{"iopub.status.busy":"2023-10-17T14:33:48.933487Z","iopub.execute_input":"2023-10-17T14:33:48.933887Z","iopub.status.idle":"2023-10-17T14:33:50.101651Z","shell.execute_reply.started":"2023-10-17T14:33:48.933816Z","shell.execute_reply":"2023-10-17T14:33:50.100108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Use a Subset of the Total Dataset for Faster Prototyping**\n<br>-Using all of the images would take a significant amount of computation time. For faster prototyping, let's initially look at the top 25 classes by frequency\n<br>-Can expand to include full 120 classes with more compute","metadata":{"_uuid":"6b11f8be64f32b1f66cebca31f4ebb70857553d4"}},{"cell_type":"code","source":"INPUT_SIZE = 224\nNUM_CLASSES = 25\nSEED = 7\n\n# Read the y-true-labels file as well as the prediction file\ndata_dir = '../input/dog-breed-identification'\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\n\nprint(\"Num training examples files: \" + str(len(listdir(join(data_dir, 'train')))) + \" | Num training examples labels: \"\n      + str(len(labels)))\nprint(\"Num test examples files: \" + str(len(listdir(join(data_dir, 'test')))) + \" | Num test examples predictions: \" \n     + str(len(sample_submission)))","metadata":{"_uuid":"09fd34de680613d6fa7e3a8b8821b977c7136eff","execution":{"iopub.status.busy":"2023-10-17T14:33:50.103502Z","iopub.execute_input":"2023-10-17T14:33:50.103793Z","iopub.status.idle":"2023-10-17T14:33:53.175217Z","shell.execute_reply.started":"2023-10-17T14:33:50.103745Z","shell.execute_reply":"2023-10-17T14:33:53.173506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\ngroup = labels.groupby(by='breed', as_index=False).agg({'id': pd.Series.nunique})\ngroup = group.sort_values('id',ascending=False)\nprint(group)\nlabels['rank'] = group['breed']\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\n\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\n\n# Train / validation split into 80%/20%\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]","metadata":{"_uuid":"1a212cb2aa006c4090b4580d33f2b19bae799650","execution":{"iopub.status.busy":"2023-10-17T14:33:53.177148Z","iopub.execute_input":"2023-10-17T14:33:53.177586Z","iopub.status.idle":"2023-10-17T14:33:53.401714Z","shell.execute_reply.started":"2023-10-17T14:33:53.177507Z","shell.execute_reply":"2023-10-17T14:33:53.400275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_img(img_id, train_or_test, size):\n    \"\"\"\n    Read and resize image.\n    # Args:\n        img_id: image filepath string\n        train_or_test: string \"train\" or \"test\"\n        size: resize the original image\n    # Returns:\n        Image as a numpy array\n    \"\"\"\n    img = image.load_img(join(data_dir, train_or_test, '%s.jpg' % img_id), target_size = size)\n    img = image.img_to_array(img)\n    return img","metadata":{"_uuid":"e07a8ab0b6c51a6bf24b03b5d2bdf37e8561c644","execution":{"iopub.status.busy":"2023-10-17T14:33:53.403420Z","iopub.execute_input":"2023-10-17T14:33:53.403698Z","iopub.status.idle":"2023-10-17T14:33:53.409645Z","shell.execute_reply.started":"2023-10-17T14:33:53.403658Z","shell.execute_reply":"2023-10-17T14:33:53.408247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Extracting Image Feature-Representations: VGG16 Network**","metadata":{"_uuid":"b72b55fd17ddc9f0ba218c6b5bf80926301bc24d"}},{"cell_type":"code","source":"INPUT_SIZE = 224\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype = 'float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Training images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","metadata":{"_uuid":"064172ba27176e638365aba802d36e3eb4677a5a","execution":{"iopub.status.busy":"2023-10-17T14:33:53.411790Z","iopub.execute_input":"2023-10-17T14:33:53.412264Z","iopub.status.idle":"2023-10-17T14:34:38.699156Z","shell.execute_reply.started":"2023-10-17T14:33:53.412179Z","shell.execute_reply":"2023-10-17T14:34:38.698157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train / validation split via index\nXtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\n\n# Extracting image representation bottleneck features (\"bf\")\nvgg_bottleneck = VGG16(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_vgg_bf = vgg_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_vgg_bf = vgg_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('VGG training set bottleneck features shape: {} size: {:,}'.format(train_vgg_bf.shape, train_vgg_bf.size))\nprint('VGG validation set bottleneck features shape: {} size: {:,}'.format(valid_vgg_bf.shape, valid_vgg_bf.size))\nprint(\"VGG bottleneck features should be a 512-dimensional vector for each image example / prediction\")","metadata":{"_uuid":"4e11321c7dbc10b32372130d6bd27985b30041a9","execution":{"iopub.status.busy":"2023-10-17T14:34:38.700842Z","iopub.execute_input":"2023-10-17T14:34:38.701484Z","iopub.status.idle":"2023-10-17T14:53:08.323578Z","shell.execute_reply.started":"2023-10-17T14:34:38.701410Z","shell.execute_reply":"2023-10-17T14:53:08.322280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Logistic Regression on Extracted Bottleneck Features: VGG16**\n<br>Note: We could have also attached a fully-connected layer onto the end of the pre-trained network.\n<br>This would have worked fine, although requiring more compute and fine-tuning vs simply taking the extracted feature-representation from a pre-trained network\n<br> and doing a logistic regression on the feature-vector.","metadata":{"_uuid":"e5ce6aa1494153a037114bf003355c31b9ca1a63"}},{"cell_type":"code","source":"# Optimizer: Limited-memory BFGS\nlogreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_vgg_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_vgg_bf)\nvalid_preds = logreg.predict(valid_vgg_bf)\n\nprint('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","metadata":{"_uuid":"b5097953a788d2f9038ee6734401e41ba78d6899","execution":{"iopub.status.busy":"2023-10-17T14:53:08.324955Z","iopub.execute_input":"2023-10-17T14:53:08.325407Z","iopub.status.idle":"2023-10-17T14:53:10.971522Z","shell.execute_reply.started":"2023-10-17T14:53:08.325331Z","shell.execute_reply":"2023-10-17T14:53:10.963431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Extracting Image Feature-Representations: Xception Network**","metadata":{"_uuid":"1593352e352debf88d88cd8f38230b4423ee9a5e"}},{"cell_type":"code","source":"INPUT_SIZE = 299\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = xception.preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint(\"Training images shape: {} size: {:,}\".format(x_train.shape, x_train.size))","metadata":{"_uuid":"b53c3a1e3df680c0919ca06dae55e904201ee89b","execution":{"iopub.status.busy":"2023-10-17T14:53:10.977213Z","iopub.execute_input":"2023-10-17T14:53:10.983172Z","iopub.status.idle":"2023-10-17T14:53:30.017129Z","shell.execute_reply.started":"2023-10-17T14:53:10.983001Z","shell.execute_reply":"2023-10-17T14:53:30.015858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\n\nxception_bottleneck = xception.Xception(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_x_bf = xception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_x_bf = xception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('Xception training bottleneck features shape: {} size: {:,}'.format(train_x_bf.shape, train_x_bf.size))\nprint('Xception validation bottleneck features shape: {} size: {:,}'.format(valid_x_bf.shape, valid_x_bf.size))\n","metadata":{"_uuid":"1d3f6626942b4cb976b037ae38e2dffbee6b7cd4","execution":{"iopub.status.busy":"2023-10-17T14:53:30.018850Z","iopub.execute_input":"2023-10-17T14:53:30.019253Z","iopub.status.idle":"2023-10-17T15:18:55.916480Z","shell.execute_reply.started":"2023-10-17T14:53:30.019185Z","shell.execute_reply":"2023-10-17T15:18:55.915254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Logistic Regression on Extracted Bottleneck Features: Xception**","metadata":{"_uuid":"f0c96de69b3d6c10853ddcd722552e071ab8a7c5"}},{"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_x_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_x_bf)\nvalid_preds = logreg.predict(valid_x_bf)\nprint(\"Validation Xception LogLoss {}\".format(log_loss(yv, valid_probs)))\nprint(\"Validation Xception Accuracy {}\".format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","metadata":{"_uuid":"023440f03205148bed0562d621b0ca495254f760","execution":{"iopub.status.busy":"2023-10-17T15:18:55.918239Z","iopub.execute_input":"2023-10-17T15:18:55.918560Z","iopub.status.idle":"2023-10-17T15:19:02.097936Z","shell.execute_reply.started":"2023-10-17T15:18:55.918504Z","shell.execute_reply":"2023-10-17T15:19:02.096120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Extracting Image Feature-Representations: Inception Network**","metadata":{"_uuid":"8ef4fbc936d51afb73c71fad698d8de1ae9916d1"}},{"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint(\"X_train.shape: \" + str(Xtr.shape))\nprint(\"y_train.shape: \" + str(ytr.shape))\nprint(\"X_val.shape: \" + str(Xv.shape))\nprint(\"y_val.shape: \" + str(yv.shape))\n\ninception_bottleneck = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_i_bf = inception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_i_bf = inception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('InceptionV3 training bottleneck features shape: {} size: {:,}'.format(train_i_bf.shape, train_i_bf.size))\nprint('InceptionV3 validation bottleneck features shape: {} size: {:,}'.format(valid_i_bf.shape, valid_i_bf.size))","metadata":{"_uuid":"5a2f6df3980975c7feeec6fecf8e5e3a92e1af72","execution":{"iopub.status.busy":"2023-10-17T15:19:02.100657Z","iopub.execute_input":"2023-10-17T15:19:02.101200Z","iopub.status.idle":"2023-10-17T15:31:19.872567Z","shell.execute_reply.started":"2023-10-17T15:19:02.101120Z","shell.execute_reply":"2023-10-17T15:31:19.871517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Logistic Regression on Extracted Bottleneck Features: Inception**","metadata":{"_uuid":"4d7a1df63761f5348893a1178b31ccc773003026"}},{"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver = 'lbfgs', random_state=SEED)\nlogreg.fit(train_i_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_i_bf)\nvalid_preds = logreg.predict(valid_i_bf)\n\nprint('Validation Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","metadata":{"_uuid":"1dc74e0f8c6d9bd14b8c7514e894e119cdeba302","execution":{"iopub.status.busy":"2023-10-17T15:31:19.874111Z","iopub.execute_input":"2023-10-17T15:31:19.874498Z","iopub.status.idle":"2023-10-17T15:31:26.349847Z","shell.execute_reply.started":"2023-10-17T15:31:19.874436Z","shell.execute_reply":"2023-10-17T15:31:26.348477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Logistic Regression on Combination of Extracted Features: [Xception + Inception]**\n<br>-Leveraging the feature-extracting capabilities of multiple pre-trained models\n<br>-Benefit of using a separate independent classifier as opposed to attaching FC-layers to each of these networks: easier ensembling","metadata":{"_uuid":"e3a6712f8fac004eced7dcd1be7357ef5dc73b84"}},{"cell_type":"code","source":"X = np.hstack([train_x_bf, train_i_bf]) # This is a array-concat function that stacks horizontally instead of vertically\nV = np.hstack([valid_x_bf, valid_i_bf])\nprint(\"Full training bottleneck features shape: {} size: {:,}\".format(X.shape, X.size))\nprint(\"Full validation bottleneck features shape: {} size: {:,}\".format(V.shape, V.size))\n\nlogreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(X, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(V)\nvalid_preds = logreg.predict(V)\nprint(\"Validation Xception+Inception LogLoss {}\".format(log_loss(yv, valid_probs)))\nprint(\"Validation Xception+Inception Accuracy {}\".format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), \n                                                                        valid_preds)))","metadata":{"_uuid":"099adde9d09587e307770b0d2007f157819ad863","execution":{"iopub.status.busy":"2023-10-17T15:31:26.352493Z","iopub.execute_input":"2023-10-17T15:31:26.353639Z","iopub.status.idle":"2023-10-17T15:31:36.324019Z","shell.execute_reply.started":"2023-10-17T15:31:26.353526Z","shell.execute_reply":"2023-10-17T15:31:36.322514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Summary and Error Checking**\n<br><br>-We observe that the Xception+Inception ensemble performs very well on the given classification problem, yielding LogLoss of <0.12 and 96.5%+ accuracy.\n<br>-Below are some instances of the model's misclassifications","metadata":{"_uuid":"c5f834a85b3b0b96410f87a4ce2058a988e9ea48"}},{"cell_type":"code","source":"valid_breeds = (yv * range(NUM_CLASSES)).sum(axis=1)\nerror_idx = (valid_breeds != valid_preds)\nfor img_id, breed, pred in zip(labels.loc[valid_idx, 'id'].values[error_idx],\n                               [selected_breed_list[int(b)] for b in valid_preds[error_idx]],\n                               [selected_breed_list[int(b)] for b in valid_breeds[error_idx]]):\n    fix, ax = plt.subplots(figsize=(5,5,))\n    img = read_img(img_id, 'train', (299,299))\n    ax.imshow(img/255)\n    ax.text(10, 250, 'Prediction: %s' % pred, color='w', backgroundcolor='r', alpha=0.8)\n    ax.text(10, 270, 'LABEL: %s' % breed, color='k', backgroundcolor='g', alpha=0.8)\n    ax.axis('off')\n    plt.show()","metadata":{"_uuid":"aacd58dbb51e39e57d224acc4cbc6258202a148f","execution":{"iopub.status.busy":"2023-10-17T15:31:36.326873Z","iopub.execute_input":"2023-10-17T15:31:36.328586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"89e99eddb52bf9e076b46e9817f5a82e50015614","trusted":true},"execution_count":null,"outputs":[]}]}