{"cells":[{"metadata":{"_uuid":"b30a120404b8e104774a292b45f0902e89acef68","_cell_guid":"657faca0-c88f-4138-8c62-cf9974e0c894"},"cell_type":"markdown","source":"# Asim Fauzi & Manh Nguyen : Project I - Increasing Successful Doggy Detection\n\n## Transfer learning with pretrained Keras models\n\nCredit Due: http://vision.stanford.edu/aditya86/ImageNetDogs/\n\nAlthough Kernel resources were increased recently we still can not train useful CNNs without GPU. The original ImageNet set has quite a few different dog classes so we can reuse CNNs with pretrained ImageNet weights. Fortunately prediction is much faster (<1s/image) making it possible to run meaningful experiments with Kaggle Kernels."},{"metadata":{"_uuid":"151b0f031d10c081017bee0831d1e276148b413b","_cell_guid":"d4c4a3a8-93af-4cd2-a95b-32a2526ac3a2","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport numpy as np\nimport pandas as pd\nimport datetime as dt\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom os import listdir, makedirs\nfrom os.path import join, exists, expanduser\nfrom tqdm import tqdm\nfrom sklearn.metrics import log_loss, accuracy_score\nfrom keras.preprocessing import image\nfrom keras.applications.vgg16 import VGG16\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications import xception\nfrom keras.applications import inception_v3\nfrom keras.applications.vgg16 import preprocess_input, decode_predictions\nfrom sklearn.linear_model import LogisticRegression","execution_count":28,"outputs":[]},{"metadata":{"_uuid":"928b27a4686daca6a69bcf74bb592c9e99393992","_cell_guid":"c60a6481-e35a-49fd-b1b3-4c74f4adc472","trusted":true},"cell_type":"code","source":"start = dt.datetime.now()","execution_count":29,"outputs":[]},{"metadata":{"_uuid":"21279ccec131b58e04cca5516041df6046693ec1","_cell_guid":"9ed0e437-74af-4568-8d15-342e9adcdb5d"},"cell_type":"markdown","source":"# Use Keras Pretrained Models dataset\n\nKernels can't use network connection to download pretrained keras model weights.\nThis dataset helps you to apply your favorite pretrained model in the Kaggle Kernel environment. \nYou can find more details [here](https://www.kaggle.com/gaborfodor/keras-pretrained-models).\n\nWe have to copy the pretrained models to the cache directory (~/.keras/models) where keras is looking for them.\n"},{"metadata":{"_uuid":"4838c6bd2565ff67ce2c47ba014035c7f70a9018","_cell_guid":"d2bdd2f5-2395-43ef-a971-5472066a0d36","trusted":true},"cell_type":"code","source":"!ls ../input/keras-pretrained-models/","execution_count":30,"outputs":[{"output_type":"stream","text":"Kuszma.JPG\r\nimagenet_class_index.json\r\ninception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5\r\ninception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\r\ninception_v3_weights_tf_dim_ordering_tf_kernels.h5\r\ninception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nresnet50_weights_tf_dim_ordering_tf_kernels.h5\r\nresnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nvgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nxception_weights_tf_dim_ordering_tf_kernels.h5\r\nxception_weights_tf_dim_ordering_tf_kernels_notop.h5\r\n","name":"stdout"}]},{"metadata":{"_uuid":"ac04695a27c65eccb786d1e48fe229a3c1e288a8","_cell_guid":"482686db-3424-4b64-86b9-119563d77940","trusted":true},"cell_type":"code","source":"cache_dir = expanduser(join('~', '.keras'))\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models')\nif not exists(models_dir):\n    makedirs(models_dir)","execution_count":31,"outputs":[]},{"metadata":{"_uuid":"463b5222d5affca14b75657257f0efeb40be4ea0","_cell_guid":"ed6bb30d-f37b-4662-a1c4-ed02974204aa","trusted":true},"cell_type":"code","source":"!cp ../input/keras-pretrained-models/*notop* ~/.keras/models/\n!cp ../input/keras-pretrained-models/imagenet_class_index.json ~/.keras/models/\n!cp ../input/keras-pretrained-models/resnet50* ~/.keras/models/","execution_count":32,"outputs":[]},{"metadata":{"_uuid":"7506ec54c661024a9636fb247f4a05473aa9982d","_cell_guid":"c357ada4-644c-403a-ba61-0a84c9510a89","trusted":true},"cell_type":"code","source":"!ls ~/.keras/models","execution_count":33,"outputs":[{"output_type":"stream","text":"imagenet_class_index.json\r\ninception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\r\ninception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nresnet50_weights_tf_dim_ordering_tf_kernels.h5\r\nresnet50_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nvgg16_weights_tf_dim_ordering_tf_kernels_notop.h5\r\nxception_weights_tf_dim_ordering_tf_kernels_notop.h5\r\n","name":"stdout"}]},{"metadata":{"_uuid":"db9e42caac1312e22a6d4b0d3eec804c74e17e16","_cell_guid":"aa2e9829-2f0e-43ef-bc30-1023bca24579","trusted":true},"cell_type":"code","source":"!ls ../input/dog-breed-identification","execution_count":34,"outputs":[{"output_type":"stream","text":"labels.csv  sample_submission.csv  test  train\r\n","name":"stdout"}]},{"metadata":{"_uuid":"48c9f0e65f26fa26375dd32289f9c59bd1353d1a","_cell_guid":"7024f870-2c6f-4b86-a64e-adab46e34c1b"},"cell_type":"markdown","source":"# Use top 16 classes\nUsing all the images would take more than the 1 hour kernel limit. Let's focus on the most frequent 16 breeds."},{"metadata":{"_uuid":"3f62e82d998e8b2ba3999542492e632c5083a901","_cell_guid":"8bcb2bda-88dc-4ad8-9177-0c126401c3e1","trusted":true},"cell_type":"code","source":"INPUT_SIZE = 224\nNUM_CLASSES = 16\nSEED = 1987\ndata_dir = '../input/dog-breed-identification'\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\nprint(len(listdir(join(data_dir, 'train'))), len(labels))\nprint(len(listdir(join(data_dir, 'test'))), len(sample_submission))","execution_count":35,"outputs":[{"output_type":"stream","text":"10223 10222\n10358 10357\n","name":"stdout"}]},{"metadata":{"collapsed":true,"_uuid":"ab322ce7a697d43a883b1725c7f71bf4486fd5ed","_cell_guid":"f55b18df-3698-4146-9157-913227416e21"},"cell_type":"markdown","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\nlabels['rank'] = labels.groupby('breed').rank()['id']\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]"},{"metadata":{"_uuid":"ab322ce7a697d43a883b1725c7f71bf4486fd5ed","_cell_guid":"f55b18df-3698-4146-9157-913227416e21","trusted":true},"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\n\n#labels['rank'] = labels.groupby('breed').rank()['id'] <-- take this part out\ngroup = labels.groupby(by='breed', as_index=False).agg({'id': pd.Series.nunique})\ngroup = group.sort_values('id',ascending=False)\nprint(group)\nlabels['rank'] = group['breed']\n\n\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\nnp.random.seed(seed=SEED)\nrnd = np.random.random(len(labels))\ntrain_idx = rnd < 0.8\nvalid_idx = rnd >= 0.8\ny_train = labels_pivot[selected_breed_list].values\nytr = y_train[train_idx]\nyv = y_train[valid_idx]","execution_count":36,"outputs":[{"output_type":"stream","text":"                   breed   id\n13    scottish_deerhound  126\n10           maltese_dog  117\n0           afghan_hound  116\n6            entlebucher  115\n4   bernese_mountain_dog  114\n14              shih-tzu  112\n7         great_pyrenees  111\n11            pomeranian  111\n2                basenji  110\n12               samoyed  109\n1               airedale  107\n15       tibetan_terrier  107\n5                  cairn  106\n9               leonberg  106\n3                 beagle  105\n8       japanese_spaniel  105\n","name":"stdout"}]},{"metadata":{"_uuid":"7d26cc67909b5bd70173b5f2ed8352b210e06fb3","_cell_guid":"c48fc864-d70f-4045-96eb-12de12c0ad41","trusted":true},"cell_type":"code","source":"def read_img(img_id, train_or_test, size):\n    \"\"\"Read and resize image.\n    # Arguments\n        img_id: string\n        train_or_test: string 'train' or 'test'.\n        size: resize the original image.\n    # Returns\n        Image as numpy array.\n    \"\"\"\n    img = image.load_img(join(data_dir, train_or_test, '%s.jpg' % img_id), target_size=size)\n    img = image.img_to_array(img)\n    return img","execution_count":37,"outputs":[]},{"metadata":{"_uuid":"3bfb773d8c0b538400978a63de6ea47d7fb5d4d5","_cell_guid":"e7ee34ac-0ad9-49dc-8558-0a3ae120b287"},"cell_type":"markdown","source":"# ResNet50 class predictions for example images"},{"metadata":{"_uuid":"e48ddfeed10002d48a318c564921c6e9e5b3c5f3","_cell_guid":"f0a4e5c1-02af-4689-9845-f0f77322fb46","trusted":true},"cell_type":"code","source":"model = ResNet50(weights='imagenet')\nj = int(np.sqrt(NUM_CLASSES))\ni = int(np.ceil(1. * NUM_CLASSES / j))\nfig = plt.figure(1, figsize=(16, 16))\ngrid = ImageGrid(fig, 111, nrows_ncols=(i, j), axes_pad=0.05)\nfor i, (img_id, breed) in enumerate(labels.loc[labels['rank'] == 1, ['id', 'breed']].values):\n    ax = grid[i]\n    img = read_img(img_id, 'train', (224, 224))\n    ax.imshow(img / 255.)\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    preds = model.predict(x)\n    _, imagenet_class_name, prob = decode_predictions(preds, top=1)[0][0]\n    ax.text(10, 180, 'ResNet50: %s (%.2f)' % (imagenet_class_name , prob), color='w', backgroundcolor='k', alpha=0.8)\n    ax.text(10, 200, 'LABEL: %s' % breed, color='k', backgroundcolor='w', alpha=0.8)\n    ax.axis('off')\nplt.show()","execution_count":38,"outputs":[{"output_type":"display_data","data":{"text/plain":"<Figure size 1152x1152 with 32 Axes>","image/png":"iVBORw0KGgoAAAANSUhEUgAAA5IAAAOJCAYAAABvYB1KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3W2Infd55/HfVTtuwE2ybDSFYkm1oUpc4RacDI5LWGqIC5JfSC8KxSoldTARtHEpJJS6JDjFpdC0kMKC8qCyJd0uiavkRaumKi4bHNInB81sUuMHVAb1wVIDkbyuaQiN4+21L+bYPZlInvOXzpGO4s8HBHPf93/OXH7x5+abc5+T6u4AAADArL7vag8AAADAtUVIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMGTbkKyq36+qr1fVkxe5XlX136tqo6qeqKq3zX9MAAAAlsUs70h+Ksm+V7m+P8meyb/DST5++WMBAACwrLYNye7+UpL/+ypLDib5n73p8ST/pap+aF4DAgAAsFzm8RnJm5I8O3V8ZnIOAACA70HXX8k/VlWHs/n4a2688ca333rrrVfyz8Nrwvr6+jeSnErsM1iU9fX180k+GPc0WBj7DK6M9fX18929Mvp78wjJs0l2TR3vnJz7Lt19NMnRJFldXe21tbU5/HlgWlWd6u7VxD6DRamqf3JPg8Wyz+DKqKp/upTfm8ejrceTvHvy7a13Jnmhu782h9cFAABgCW37jmRVfSbJXUl2VNWZJB9O8rok6e5PJDmR5J4kG0m+meQ9ixoWAACAq2/bkOzuQ9tc7yTvm9tEAAAALLV5PNoKAADAa4iQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYMhMIVlV+6rqVFVtVNWDF7i+u6oeq6qvVNUTVXXP/EcFAABgGWwbklV1XZIjSfYn2ZvkUFXt3bLsQ0mOdfftSe5N8rF5DwoAAMBymOUdyTuSbHT36e5+MckjSQ5uWdNJ3jj5+U1J/mV+IwIAALBMrp9hzU1Jnp06PpPkHVvW/HqSv6iqX0pyY5K75zIdAAAAS2deX7ZzKMmnuntnknuS/GFVfddrV9XhqlqrqrVz587N6U8DW+ywz2Dx3NNg8ewzWF6zhOTZJLumjndOzk27P8mxJOnuv03y+iQ7tr5Qdx/t7tXuXl1ZWbm0iYHtnLfPYPHc02Dx7DNYXrOE5Mkke6rqlqq6IZtfpnN8y5p/TvKuJKmqH81mSPqfjQAAAL4HbRuS3f1SkgeSPJrkmWx+O+tTVfVwVR2YLPtAkvdW1d8l+UyS+7q7FzU0AAAAV88sX7aT7j6R5MSWcw9N/fx0knfOdzQAAACW0by+bAcAAIDXCCEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAECEJAADAkJlCsqr2VdWpqtqoqgcvsuZnqurpqnqqqj493zEBAABYFtdvt6CqrktyJMlPJTmT5GRVHe/up6fW7Enya0ne2d3PV9UPLmpgAAAArq5Z3pG8I8lGd5/u7heTPJLk4JY1701ypLufT5Lu/vp8xwQAAGBZzBKSNyV5dur4zOTctLckeUtV/XVVPV5V++Y1IAAAAMtl20dbB15nT5K7kuxM8qWq+rHu/tfpRVV1OMnhJNm9e/ec/jSwxY6qWkvsM1gk9zRYPPsMltcs70ieTbJr6njn5Ny0M0mOd/e3u/sfkvx9NsPyO3T30e5e7e7VlZWVS50ZeHXn7TNYPPc0WDz7DJbXLCF5Msmeqrqlqm5Icm+S41vW/HE2341MVe3I5qOup+c4JwAAAEti25Ds7peSPJDk0STPJDnW3U9V1cNVdWCy7NEkz1XV00keS/Ir3f3cooYGAADg6pnpM5LdfSLJiS3nHpr6uZO8f/IPAACA72GzPNoKAAAArxCSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADBGSAAAADJkpJKtqX1WdqqqNqnrwVdb9dFV1Va3Ob0QAAACWybYhWVXXJTmSZH+SvUkOVdXeC6x7Q5JfTvLleQ8JAADA8pjlHck7kmx09+nufjHJI0kOXmDdbyT5SJJ/n+N8AAAALJlZQvKmJM9OHZ+ZnHtFVb0tya7u/rM5zgYAAMASuuwv26mq70vy0SQfmGHt4apaq6q1c+fOXe6fBi5sh30Gi+eeBotnn8HymiUkzybZNXW8c3LuZW9IcluSL1bVPya5M8nxC33hTncf7e7V7l5dWVm59KmBV3PePoPFc0+DxbPPYHnNEpInk+ypqluq6oYk9yY5/vLF7n6hu3d0983dfXOSx5Mc6O61hUwMAADAVbVtSHb3S0keSPJokmeSHOvup6rq4ao6sOgBAQAAWC7Xz7Kou08kObHl3EMXWXvX5Y8FAADAsrrsL9sBAADgtUVIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMERIAgAAMGSmkKyqfVV1qqo2qurBC1x/f1U9XVVPVNUXquqH5z8qAAAAy2DbkKyq65IcSbI/yd4kh6pq75ZlX0my2t0/nuRzSX573oMCAACwHGZ5R/KOJBvdfbq7X0zySJKD0wu6+7Hu/ubk8PEkO+c7JgAAAMtilpC8KcmzU8dnJucu5v4kf345QwEAALC8rp/ni1XVzyVZTfKTF7l+OMnhJNm9e/c8/zTwn3ZU1Vpin8EiuafB4tlnsLxmeUfybJJdU8c7J+e+Q1XdneSDSQ5097cu9ELdfbS7V7t7dWVl5VLmBbZ33j6DxXNPg8Wzz2B5zRKSJ5PsqapbquqGJPcmOT69oKpuT/LJbEbk1+c/JgAAAMti25Ds7peSPJDk0STPJDnW3U9V1cNVdWCy7HeS/ECSz1bVV6vq+EVeDgAAgGvcTJ+R7O4TSU5sOffQ1M93z3kuAAAAltQsj7YCAADAK4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ4QkAAAAQ2YKyaraV1Wnqmqjqh68wPXvr6o/mlz/clXdPO9BAQAAWA7bhmRVXZfkSJL9SfYmOVRVe7csuz/J8939I0l+N8lH5j0oAAAAy2GWdyTvSLLR3ae7+8UkjyQ5uGXNwSR/MPn5c0neVVU1vzEBAABYFrOE5E1Jnp06PjM5d8E13f1SkheSvHkeAwIAALBcrp9hzS8m+YmqurO7b9t6cfLO4w8l+cuq+rck913sharqcJLDk8NvVdWT4yNfFTuSnL/aQ8zIrItxLc16W1WtTX62zxbDrItxLc36Vve0K8Ksi3GtzGqfXRlmXYxrada3XsovzRKSn03y+iRvnBzvTHJ26vr+JP+R5GeTdJKPJ3lTkue2vlB3H01yNEmqaq27Vy9l6CvNrIth1sWYnvVanXvZmXUxrsFZ3dMWzKyLca3Map9dGWZdjGtt1kv5vVkebf29bMbj66rqhiT3Jjk+df1gkj9N8vPd/XiSXUn+prv7UgYCAABgudUsvVdV9yX5ZDY/H/n73f2bVfVwkrVsPm7w0SS/kOT2JCtJ3t3df3KB13nl8YQbb7zx7bfeeuuc/jOAl62vr/+/JF9N7DNYlPX19ZeSvC/uabAw9hlcGevr6y919+tGf2/WkLw5yecv8hnJzyf5re7+q8nxF5L8ane/6lukq6urvbZ2Se+iAq+iqv6pu29O7DNYlOl9lthrsAj2GVwZW/farGZ5tHU7Z7P5OOvLtn6GEriyrpUPdsO1zD6DxbPP4Mq4pL02j5A8nuTdtenOJC9099fm8LoAAAAsoW2/tbWqPpPkriQ7qupMkg8neV2SdPcnkpxIck+SjSTfTPKeRQ0LAADA1bdtSHb3oW2udzY/CA0AAMBrwDwebQUAAOA1REgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwREgCAAAwZKaQrKp9VXWqqjaq6sELXN9dVY9V1Veq6omqumf+owIAALAMtg3JqrouyZEk+5PsTXKoqvZuWfahJMe6+/Yk9yb52LwHBQAAYDnM8o7kHUk2uvt0d7+Y5JEkB7es6SRvnPz8piT/Mr8RAQAAWCbXz7DmpiTPTh2fSfKOLWt+PclfVNUvJbkxyd1zmQ4AAIClM68v2zmU5FPdvTPJPUn+sKq+67Wr6nBVrVXV2rlz5+b0p4EtdthnsHjuabB49hksr1lC8mySXVPHOyfnpt2f5FiSdPffJnl9kh1bX6i7j3b3anevrqysXNrEwHbO22eweO5psHj2GSyvWULyZJI9VXVLVd2QzS/TOb5lzT8neVeSVNWPZjMk/c9GAAAA34O2DcnufinJA0keTfJMNr+d9amqeriqDkyWfSDJe6vq75J8Jsl93d2LGhoAAICrZ5Yv20l3n0hyYsu5h6Z+fjrJO+c7GgAAAMtoXl+2AwAAwGuEkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGDITCFZVfuq6lRVbVTVgxdZ8zNV9XRVPVVVn57vmAAAACyL67dbUFXXJTmS5KeSnElysqqOd/fTU2v2JPm1JO/s7uer6gcXNTAAAABX1yzvSN6RZKO7T3f3i0keSXJwy5r3JjnS3c8nSXd/fb5jAgAAsCxmCcmbkjw7dXxmcm7aW5K8par+uqoer6p98xoQAACA5bLto60Dr7MnyV1Jdib5UlX9WHf/6/Siqjqc5HCS7N69e05/GthiR1WtJfYZLJJ7GiyefQbLa5Z3JM8m2TV1vHNybtqZJMe7+9vd/Q9J/j6bYfkduvtod6929+rKysqlzgy8uvP2GSyeexosnn0Gy2uWkDyZZE9V3VJVNyS5N8nxLWv+OJvvRqaqdmTzUdfTc5wTAACAJbFtSHb3S0keSPJokmeSHOvup6rq4ao6MFn2aJLnqurpJI8l+ZXufm5RQwMAAHD1zPQZye4+keTElnMPTf3cSd4/+QcAAMD3sFkebQUAAIBXCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGCEkAAACGzBSSVbWvqk5V1UZVPfgq6366qrqqVuc3IgAAAMtk25CsquuSHEmyP8neJIeqau8F1r0hyS8n+fK8hwQAAGB5zPKO5B1JNrr7dHe/mOSRJAcvsO43knwkyb/PcT4AAACWzCwheVOSZ6eOz0zOvaKq3pZkV3f/2RxnAwAAYAld9pftVNX3Jflokg/MsPZwVa1V1dq5c+cu908DF7bDPoPFc0+DxbPPYHnNEpJnk+yaOt45OfeyNyS5LckXq+ofk9yZ5PiFvnCnu49292p3r66srFz61MCrOW+fweK5p8Hi2WewvGYJyZNJ9lTVLVV1Q5J7kxx/+WJ3v9DdO7r75u6+OcnjSQ5099pCJgYAAOCq2jYku/ulJA8keTTJM0mOdfdTVfVwVR1Y9IAAAAAsl+tnWdTdJ5Kc2HLuoYusvevyxwIAAGBZXfaX7QAAAPDaIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYMlNIVtW+qjpVVRtV9eAFrr+/qp6uqieq6gtV9cPzHxUAAIBlsG1IVtV1SY4k2Z9kb5JDVbV3y7KvJFnt7h9P8rkkvz3vQQEAAFgOs7wjeUeSje4+3d0vJnkkycHpBd39WHd/c3L4eJKd8x0TAACAZTFLSN6U5Nmp4zOTcxdzf5I/v5yhAAAAWF7Xz/PFqurnkqwm+cmLXD+c5HCS7N69e55/GvhPO6pqLbHPYJHc02Dx7DNYXrO8I3k2ya6p452Tc9+hqu5O8sEkB7r7Wxd6oe4+2t2r3b26srJyKfMC2ztvn8HiuafB4tlnsLxmCcmTSfZU1S1VdUOSe5Mcn15QVbcn+WQ2I/Lr8x8TAACAZbFtSHb3S0keSPJokmeSHOvup6rq4ao6MFn2O0l+IMlnq+qrVXX8Ii8HAADANW6mz0h294kkJ7ace2jq57vnPBcAAABLapZHWwEAAOAVQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhQhIAAIAhM4VkVe2rqlNVtVFVD17g+vdX1R9Nrn+5qm6e96AAAAAsh21DsqquS3Ikyf4ke5Mcqqq9W5bdn+T57v6RJL+b5CPzHhQAAIDlMMs7knck2eju0939YpJHkhzcsuZgkj+Y/Py5JO+qqprfmAAAACyLWULypiTPTh2fmZy74JrufinJC0nePI8BAQAAWC7Xz7DmF5P8RFXd2d23bb04eefxh5L8ZVX9W5L7LvZCVXU4yeHJ4beq6snxka+KHUnOX+0hZmTWxbiWZr2tqtYmP9tni2HWxbiWZn2re9oVYdbFuFZmtc+uDLMuxrU061sv5ZdmCcnPJnl9kjdOjncmOTt1fX+S/0jys0k6yceTvCnJc1tfqLuPJjmaJFW11t2rlzL0lWbWxTDrYkzPeq3OvezMuhjX4KzuaQtm1sW4Vma1z64Msy7GtTbrpfzeLI+2/l424/F1VXVDknuTHJ+6fjDJnyb5+e5+PMmuJH/T3X0pAwEAALDctg3JyWceH0pyc5Jnkhzr7qeq6uGqOpDNz0f+ryRvrqqNJDcm+R+LGxkAAICrqWZ543Dy/wv5+Yt8RvLzSX6ru/9qcvyFJL/a3d/1Fun0c+433njj22+99dbLGh74buvr699K8mRin8GirK+vfyPJB+KeBgtjn8GVsb6+/o3ufsPo783yGcntnM3m46wv2/oZyldMP+e+urraa2uX9Dgu8Cqq6smXn8m3z2AxquqUexosln0GV0ZVnbqU35vlM5LbOZ7k3bXpziQvdPfX5vC6AAAALKFt35Gsqs8kuSvJjqo6k+TDSV6XJN39iSQnktyTZCPJN5O8Z1HDAgAAcPVtG5LdfWib653kfXObCAAAgKU2j0dbAQAAeA0RkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAwRkgAAAAyZKSSral9Vnaqqjap68ALXd1fVY1X1lap6oqrumf+oAAAALINtQ7KqrktyJMn+JHuTHKqqvVuWfSjJse6+Pcm9ST4270EBAABYDrO8I3lHko3uPt3dLyZ5JMnBLWs6yRsnP78pyb/Mb0QAAACWyfUzrLkpybNTx2eSvGPLml9P8hdV9UtJbkxy91ymAwAAYOnM68t2DiX5VHfvTHJPkj+squ967ao6XFVrVbV27ty5Of1pYIsd9hksnnsaLJ59BstrlpA8m2TX1PHOyblp9yc5liTd/bdJXp9kx9YX6u6j3b3a3asrKyuXNjGwnfP2GSyeexosnn0Gy2uWkDyZZE9V3VJVN2Tzy3SOb1nzz0nelSRV9aPZDEn/sxEAAMD3oG1DsrtfSvJAkkeTPJPNb2d9qqoerqoDk2UfSPLeqvq7JJ9Jcl9396KGBgAA4OqZ5ct20t0nkpzYcu6hqZ+fTvLO+Y4GAADAMprXl+0AAADwGiEkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGCIkAQAAGDJTSFbVvqo6VVUbVfXgRdb8TFU9XVVPVdWn5zsmAAAAy+L67RZU1XVJjiT5qSRnkpysquPd/fTUmj1Jfi3JO7v7+ar6wUUNDAAAwNU1yzuSdyTZ6O7T3f1ikkeSHNyy5r1JjnT380nS3V+f75gAAAAsi1lC8qYkz04dn5mcm/aWJG+pqr+uqserat+8BgQAAGC5bPto68Dr7ElyV5KdSb5UVT/W3f86vaiqDic5nCS7d++e058GtthRVWuJfQaL5J4Gi2efwfKa5R3Js0l2TR3vnJybdibJ8e7+dnf/Q5K/z2ZYfofuPtrdq929urKycqkzA6/uvH0Gi+eeBotnn8HymiUkTybZU1W3VNUNSe5NcnzLmj/O5ruRqaod2XzU9fQc5wQAAGBJbBuS3f1SkgeSPJrkmSTHuvupqnq4qg5Mlj2a5LmqejrJY0l+pbufW9TQAAAAXD0zfUayu08kObHl3ENTP3eS90/+AQAA8D1slkdbAQAA4BVCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCFCEgAAgCEzhWRV7auqU1W1UVUPvsq6n66qrqrV+Y0IAADAMtk2JKvquiRHkuxPsjfJoarae4F1b0jyy0m+PO8hAQAAWB6zvCN5R5KN7j7d3S8meSTJwQus+40kH0ny73OcDwAAgCUzS0jelOTZqeMzk3OvqKq3JdnV3X82x9kAAABYQpf9ZTtV9X1JPprkAzOsPVxVa1W1du7cucv908CF7bDPYPHc02Dx7DNYXrOE5Nkku6aOd07OvewNSW5L8sWq+sckdyY5fqEv3Onuo9292t2rKysrlz418GrO22eweO5h2sLXAAAWt0lEQVRpsHj2GSyvWULyZJI9VXVLVd2Q5N4kx1++2N0vdPeO7r65u29O8niSA929tpCJAQAAuKq2DcnufinJA0keTfJMkmPd/VRVPVxVBxY9IAAAAMvl+lkWdfeJJCe2nHvoImvvuvyxAAAAWFaX/WU7AAAAvLYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIYISQAAAIbMFJJVta+qTlXVRlU9eIHr76+qp6vqiar6QlX98PxHBQAAYBlsG5JVdV2SI0n2J9mb5FBV7d2y7CtJVrv7x5N8Lslvz3tQAAAAlsMs70jekWSju09394tJHklycHpBdz/W3d+cHD6eZOd8xwQAAGBZzBKSNyV5dur4zOTcxdyf5M8vZygAAACW1/XzfLGq+rkkq0l+8iLXDyc5nCS7d++e558G/tOOqlpL7DNYJPc0WDz7DJbXLO9Ink2ya+p45+Tcd6iqu5N8MMmB7v7WhV6ou49292p3r66srFzKvMD2zttnsHjuabB49hksr1lC8mSSPVV1S1XdkOTeJMenF1TV7Uk+mc2I/Pr8xwQAAGBZbBuS3f1SkgeSPJrkmSTHuvupqnq4qg5Mlv1Okh9I8tmq+mpVHb/IywEAAHCNm+kzkt19IsmJLecemvr57jnPBQAAwJKa5dFWAAAAeIWQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYIiQBAAAYMhMIVlV+6rqVFVtVNWDF7j+/VX1R5PrX66qm+c9KAAAAMth25CsquuSHEmyP8neJIeqau+WZfcneb67fyTJ7yb5yLwHBQAAYDnM8o7kHUk2uvt0d7+Y5JEkB7esOZjkDyY/fy7Ju6qq5jcmAAAAy+L6Gdb8ZpKfqKonu/u2JGeSvOPli5NgXE3y6ar6tyT3JXkhyZuTnJ9+oao6nOTw5PBbVfXkZf8XXBk7suW/ZYmZdTGupVlvq6q1yc/22WKYdTGupVnf6p52RZh1Ma6VWe2zK8Osi3EtzfrWS/mlWULysWyG4Z6LXN+f5IYk/y3JziQfv9gLdffRJEeTpKrWunt1aNqrxKyLYdbFmJ71Wp172Zl1Ma7BWd3TFsysi3GtzGqfXRlmXYxrbdZL+b1ZHm3930n+69TxziRnp44PJjmdZFd3P57kv0z+PXcpAwEAALDcZgnJk0luSfK6qrohyb1Jjk9dvymbsfnzk+NvJ/k/3d3zHBQAAIDlULP0XlXdl+ST2fx85O93929W1cNJ1rL53PpHk/xCktuTrCR5d3f/yQVe55Xn3G+88ca333rrrXP6zwBetr6+/q0kTyb2GSzK+vr6N5J8IO5psDD2GVwZ6+vr3+juN4z+3qwheXOSz0++bGfrtU8m+WJ3f2ZyfCrJXd39tVd7zdXV1V5bu6THcYFXUVXrLz+Tb5/BYkzvs8Reg0Wwz+DK2LrXZjXLo63bOZ7k3bXpziQvbBeRAAAAXLu2/dbWqvpMkruS7KiqM0k+nOR1SdLdn0hyIsk9STaSfDPJexY1LAAAAFfftiHZ3Ye2ud5J3je3iQAAAFhq83i0FQAAgNcQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMAQIQkAAMCQmUKyqvZV1amq2qiqBy9wfXdVPVZVX6mqJ6rqnvmPCgAAwDLYNiSr6rokR5LsT7I3yaGq2rtl2YeSHOvu25Pcm+Rj8x4U/n979xdq2VmeAfx5TYjSNtriTKBkEhPpRJzGgmGItkK1mJZoIXNhKwkIlQZTbSMFpZAiBIlXttSWlkA7UNEKGqMXZagjKbWRFDExA/5NQmQcQzOxNJM0eiMag28v9k482TmTs7+Zvc7ZJ/5+cGCttb/Z58me856VZ/9ZAwAArIdlXpG8Msnx7j7R3U8muS3JoYU1neSl8+2XJfne6iICAACwTs5dYs2FSR7esH8yyesW1nwwyb9X1XuT/GKSq1aSDgAAgLWzqovtXJfkY929L8lbk3yiqp5z31V1Q1Udq6pjp06dWtG3BhbsMWcwPec0mJ45g/W1TJF8JMlFG/b3zY9tdH2S25Oku7+c5CVJ9izeUXcf7u6D3X1w7969Z5YY2Mpj5gym55wG0zNnsL6WKZL3JtlfVZdW1XmZXUznyMKa/07y5iSpqldnViQ9bQQAAPACtGWR7O6nktyY5I4kD2R2ddb7quqWqrpmvuz9Sd5VVV9P8qkk7+zunio0AAAAO2eZi+2ku48mObpw7OYN2/cnecNqowEAALCOVnWxHQAAAH5OKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhiiSAAAADFEkAQAAGKJIAgAAMESRBAAAYIgiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADBEkQQAAGCIIgkAAMAQRRIAAIAhiiQAAABDFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhiiSAAAADFmqSFbV1VX1YFUdr6qbTrPm7VV1f1XdV1WfXG1MAAAA1sW5Wy2oqnOS3Jrkd5OcTHJvVR3p7vs3rNmf5C+TvKG7n6iqC6YKDAAAwM5a5hXJK5Mc7+4T3f1kktuSHFpY864kt3b3E0nS3Y+uNiYAAADrYpkieWGShzfsn5wf2+iyJJdV1Zeq6u6qunpVAQEAAFgvW761deB+9id5U5J9Se6qqtd09/c3LqqqG5LckCQXX3zxir41sGBPVR1LzBlMyTkNpmfOYH0t84rkI0ku2rC/b35so5NJjnT3T7r7u0m+nVmxfJbuPtzdB7v74N69e880M/D8HjNnMD3nNJieOYP1tUyRvDfJ/qq6tKrOS3JtkiMLa/41s1cjU1V7Mnur64kV5gQAAGBNbFkku/upJDcmuSPJA0lu7+77quqWqrpmvuyOJI9X1f1J7kzyF939+FShAQAA2DlLfUayu48mObpw7OYN253kffMvAAAAXsCWeWsrAAAAPEORBAAAYIgiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADBEkQQAAGCIIgkAAMAQRRIAAIAhiiQAAABDFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhiiSAAAADFEkAQAAGKJIAgAAMESRBAAAYIgiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADBEkQQAAGDIUkWyqq6uqger6nhV3fQ8695WVV1VB1cXEQAAgHWyZZGsqnOS3JrkLUkOJLmuqg5ssu78JH+e5J5VhwQAAGB9LPOK5JVJjnf3ie5+MsltSQ5tsu5DST6c5EcrzAcAAMCaWaZIXpjk4Q37J+fHnlFVVyS5qLs/t8JsAAAArKGzvthOVb0oyUeSvH+JtTdU1bGqOnbq1Kmz/dbA5vaYM5iecxpMz5zB+lqmSD6S5KIN+/vmx552fpLLk3yxqh5K8vokRza74E53H+7ug919cO/evWeeGng+j5kzmJ5zGkzPnMH6WqZI3ptkf1VdWlXnJbk2yZGnb+zuH3T3nu6+pLsvSXJ3kmu6+9gkiQEAANhRWxbJ7n4qyY1J7kjyQJLbu/u+qrqlqq6ZOiAAAADr5dxlFnX30SRHF47dfJq1bzr7WAAAAKyrs77YDgAAAD9fFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhiiSAAAADFEkAQAAGKJIAgAAMESRBAAAYIgiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADBEkQQAAGCIIgkAAMAQRRIAAIAhiiQAAABDFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhixVJKvq6qp6sKqOV9VNm9z+vqq6v6q+UVVfqKpXrD4qAAAA62DLIllV5yS5NclbkhxIcl1VHVhY9tUkB7v7N5J8NslfrTooAAAA62GZVySvTHK8u09095NJbktyaOOC7r6zu3843707yb7VxgQAAGBdLFMkL0zy8Ib9k/Njp3N9ks+fTSgAAADW17mrvLOqekeSg0neeJrbb0hyQ5JcfPHFq/zWwM/sqapjiTmDKTmnwfTMGayvZV6RfCTJRRv2982PPUtVXZXkA0mu6e4fb3ZH3X24uw9298G9e/eeSV5ga4+ZM5iecxpMz5zB+lqmSN6bZH9VXVpV5yW5NsmRjQuq6rVJ/imzEvno6mMCAACwLrYskt39VJIbk9yR5IEkt3f3fVV1S1VdM1/210l+KclnquprVXXkNHcHAADALrfUZyS7+2iSowvHbt6wfdWKcwEAALCmlnlrKwAAADxDkQQAAGCIIgkAAMAQRRIAAIAhiiQAAABDFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgiCIJAADAEEUSAACAIYokAAAAQxRJAAAAhiiSAAAADFEkAQAAGKJIAgAAMESRBAAAYIgiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADBEkQQAAGCIIgkAAMAQRRIAAIAhiiQAAABDFEkAAACGKJIAAAAMUSQBAAAYokgCAAAwRJEEAABgyFJFsqqurqoHq+p4Vd20ye0vrqpPz2+/p6ouWXVQAAAA1sOWRbKqzklya5K3JDmQ5LqqOrCw7PokT3T3ryX52yQfXnVQAAAA1sMyr0hemeR4d5/o7ieT3Jbk0MKaQ0k+Pt/+bJI3V1WtLiYAAADrYpkieWGShzfsn5wf23RNdz+V5AdJXr6KgAAAAKyXc7fzm1XVDUlumO/+uKq+tZ3f/yzsSfLYTodYkqzT2E1ZL6+qY/NtczYNWaexm7K+yjltW8g6jd2S1ZxtD1mnsZuyvupM/tAyRfKRJBdt2N83P7bZmpNVdW6SlyV5fPGOuvtwksNJUlXHuvvgmYTebrJOQ9ZpbMy6W3OvO1mnsQuzOqdNTNZp7Jas5mx7yDqN3Zb1TP7cMm9tvTfJ/qq6tKrOS3JtkiMLa44k+aP59h8k+c/u7jMJBAAAwHrb8hXJ7n6qqm5MckeSc5J8tLvvq6pbkhzr7iNJ/jnJJ6rqeJL/y6xsAgAA8AK01Gcku/tokqMLx27esP2jJH84+L0PD67fSbJOQ9ZpHD7N9rqTdRqyTmMx627Ovs5kncZuyWrOtoes03jBZy3vQAUAAGDEMp+RBAAAgGdMXiSr6uqqerCqjlfVTZvc/uKq+vT89nuq6pKpM53OElnfV1X3V9U3quoLVfWKncg5z/K8WTese1tVdVXt2FWjlslaVW+fP7b3VdUntzvjhhxb/QxcXFV3VtVX5z8Hb92hnB+tqkefvgz6Yu6a+fv5/jeq6nXmbJw5m8ZunbP5scVZ2zhnV6zLOc2cTcOcTcM5bXuYtWnsllnb7Jy2cPvinF2x5Z1292RfmV2c5ztJXpnkvCRfT3JgYc2fJvnH+fa1ST49ZaazzPo7SX5hvv2edc46X3d+kruS3J3k4LpmTbI/yVeT/Mp8/4I1zno4yXvm2weSPLRDWX87yRVJvnWa3H+S5PNJKsnrk3zXnK0+63ydOVt91rWbs9NkPzH/u396zu7JGpzTzNmOPq7m7MzyOqetQdb5OrO2+qxrMWtZOKdtcvtbF+bsnq3uc+pXJK9Mcry7T3T3k0luS3JoYc2hJB+fb382yZurqibOtZkts3b3nd39w/nu3Zn9m5o7YZnHNUk+lOTDSX60neEWLJP1XUlu7e4nkqS7H93mjE9bJmsneel8+2VJvreN+X4WovuuzK6QnGye+4+T/EvP3J3kgiT/Nl9vzpZjzqaxW+cseW72U0ke2TBnv5zZhed2+pxmzqZhzibinLYtzNo0ds2sbXJOW3Qoz56zX66qX32++5y6SF6Y5OEN+yfnxzZd091PJflBkpdPnGszy2Td6PrMWvtO2DLr/OXoi7r7c9sZbBPLPK6XJbmsqr5UVXdX1dXblu7Zlsn6wSTvqKqTmV3J+L3bE+15bZb7goVjneSniTkbYM6msVvnLHlu9vPmX087meSi7Pw5zZxNw5xtD+e0aZi1aezmWVs0+vO83D//wbNV1TuSHEzyxp3OspmqelGSjyR55w5HWda5mb1F4U2ZPVN3V1W9pru/v6OpNnddko91999U1W9m9u+nXt7dP93pYC805mzlzBnPYc5WzpyxKbO2cmZtDUz9iuQjmT07+7R982ObrqmqczN7yffxiXNtZpmsqaqrknwgyTXd/eNtyrZoq6znJ7k8yRer6qHM3ud8ZIc+NL3M43oyyZHu/kl3fzfJtzP75bDdlsl6fZLbk6S7v5zkJUn2bEu609ss96MLxyrzeTdnSzNn09itc5Y8N/uT86+n7cvs2dydPqeZs2mYs+3hnDYNszaN3Txri5b6eX6WrT5EeTZfmT1bcCLJpfnZB1B/fWHNn+XZH5i+fcpMZ5n1tZl9oHb/TmQcybqw/ovZuQ9ML/O4Xp3k4/PtPZn9j9jL1zTr55O8c7796sze51479NhektmFCTbL/Z48+wPTD5mz1WddWG/OVpd17ebsNNlPJPmvDXP2lazBOc2c7ejjas7OPPMlcU7b0awL683a6rKuzaxlwzltk9t+f2HOvrLl/W1D4Ldm9izBd5J8YH7slsyelUlmrfwzSY5ndhJ+5U48sEtm/Y8k/5vka/OvI+uadWHtjv0yWPJxrczeTnF/km8muXaNsx5I8qX5L4qvJfm9Hcr5qST/k+QnmT0r93fzn81TmT3rWUnuna/5ZpLfMmerz7qw1pytLuu6ztn1Sf5h/vP5nfms3ZrZxQseyuxta2txTjNnO/a4mrMzy+qctgZZF9aatdVlXYtZ22TOrk/y7iTv3vCY3jr/7/jmMn//Nf+DAAAAsJSpPyMJAADAC4wiCQAAwBBFEgAAgCGKJAAAAEMUSQAAAIYokgAAAAxRJAEAABiiSAIAADDk/wESyyfFIaAknwAAAABJRU5ErkJggg==\n"},"metadata":{}}]},{"metadata":{"_uuid":"3774760723fa28d44b3505c2047c1a0b3f32eb05","_cell_guid":"0a2cd723-1859-4b36-b5af-897b7b501c9e"},"cell_type":"markdown","source":"# Extracting Resnet50 bottleneck features\nPreprocessing and prediction seems to be working. 75% accuracy on these 16 images."},{"metadata":{"trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\ntrain_rn_bf = model.predict(Xtr, batch_size=32, verbose=1)\nvalid_rn_bf = model.predict(Xv, batch_size=32, verbose=1)","execution_count":48,"outputs":[{"output_type":"stream","text":"1409/1409 [==============================] - 7s 5ms/step\n368/368 [==============================] - 1s 3ms/step\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"# LogReg on Resnet50 bottleneck features"},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_rn_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_rn_bf)\nvalid_preds = logreg.predict(valid_rn_bf)\n\nprint('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":49,"outputs":[{"output_type":"stream","text":"Validation VGG LogLoss 0.37265557858369197\nValidation VGG Accuracy 0.9293478260869565\n","name":"stdout"}]},{"metadata":{"_uuid":"dd270f61ff0278c9171592f4999513b460f8f262","_cell_guid":"5759352a-e324-468c-b0b9-b360962a823f"},"cell_type":"markdown","source":"# Extract VGG16 bottleneck features"},{"metadata":{"_uuid":"db4f7052673aa28ca7822b9030dba078a6afb878","_cell_guid":"62de53fc-18b6-45f4-8ec3-14a3287b2015","trusted":true},"cell_type":"code","source":"INPUT_SIZE = 224\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","execution_count":41,"outputs":[{"output_type":"stream","text":"1777it [00:08, 218.69it/s]","name":"stderr"},{"output_type":"stream","text":"Train Images shape: (1777, 224, 224, 3) size: 267,488,256\n","name":"stdout"},{"output_type":"stream","text":"\n","name":"stderr"}]},{"metadata":{"_uuid":"11bfa7db44dc81846a939f2c5259b283bbb7f9e4","_cell_guid":"ff0176f1-60a6-4487-9df4-438f3604e4b8","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\nvgg_bottleneck = VGG16(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_vgg_bf = vgg_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_vgg_bf = vgg_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('VGG train bottleneck features shape: {} size: {:,}'.format(train_vgg_bf.shape, train_vgg_bf.size))\nprint('VGG valid bottleneck features shape: {} size: {:,}'.format(valid_vgg_bf.shape, valid_vgg_bf.size))","execution_count":43,"outputs":[]},{"metadata":{"_uuid":"8adbc32293a65ff184b659282e3c9a4a5eba64af","_cell_guid":"68b9963a-e2cd-463c-8708-11d2685e0eb2"},"cell_type":"markdown","source":"# LogReg on VGG bottleneck features"},{"metadata":{"_uuid":"6af396744a3c6d41f8256f993e60389d67e2ab52","_cell_guid":"90569f5c-b196-4104-9089-43d8b8ddd12b","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_vgg_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_vgg_bf)\nvalid_preds = logreg.predict(valid_vgg_bf)","execution_count":45,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n  \"of iterations.\", ConvergenceWarning)\n","name":"stderr"}]},{"metadata":{"_uuid":"4213c70e4a252cfae11f792aad256664258bb6be","_cell_guid":"c92cf2e3-bd0f-44f4-932a-8fec12bf95ea","trusted":true},"cell_type":"code","source":"print('Validation VGG LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation VGG Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":46,"outputs":[{"output_type":"stream","text":"Validation VGG LogLoss 0.3521862347494172\nValidation VGG Accuracy 0.9184782608695652\n","name":"stdout"}]},{"metadata":{"_uuid":"0c150ebe0b4cc2fe05acb28d0f5cb0b69ceaf730","_cell_guid":"dcf8bcd6-c73e-4677-a2a3-30bc1708dd96"},"cell_type":"markdown","source":"Not bad, 90% accuracy for the top 16 classes. The multiclass classification with 120 classes is more difficult so these LogLoss/Accuracy scores does not translate to LB."},{"metadata":{"_uuid":"e8f19dc62979a04aaa396f9cd9b990751d372286","_cell_guid":"c9e29a84-5cad-49d7-9290-5f7755cb1d43"},"cell_type":"markdown","source":"# Extract Xception bottleneck features"},{"metadata":{"_uuid":"7e5f115592134a49b4997903ccc9e7497797be1e","_cell_guid":"fe14c29b-42a0-45cb-a7a0-5f201c984ff6","trusted":true},"cell_type":"code","source":"INPUT_SIZE = 299\nPOOLING = 'avg'\nx_train = np.zeros((len(labels), INPUT_SIZE, INPUT_SIZE, 3), dtype='float32')\nfor i, img_id in tqdm(enumerate(labels['id'])):\n    img = read_img(img_id, 'train', (INPUT_SIZE, INPUT_SIZE))\n    x = xception.preprocess_input(np.expand_dims(img.copy(), axis=0))\n    x_train[i] = x\nprint('Train Images shape: {} size: {:,}'.format(x_train.shape, x_train.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1508b3285ea437bf24aa765c76ea56042d406034","_cell_guid":"37a9f47a-9a17-43cd-99b3-7f89192ccf34","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\nxception_bottleneck = xception.Xception(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_x_bf = xception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_x_bf = xception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('Xception train bottleneck features shape: {} size: {:,}'.format(train_x_bf.shape, train_x_bf.size))\nprint('Xception valid bottleneck features shape: {} size: {:,}'.format(valid_x_bf.shape, valid_x_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1bdd346837a8ad7fbbe1f9b5cb424a33055117d","_cell_guid":"065777fc-0308-4526-823a-4915cd5024be"},"cell_type":"markdown","source":"# LogReg on Xception bottleneck features\n"},{"metadata":{"_uuid":"db34ba100fb8a524487ed0b91134b6e65102e3fb","_cell_guid":"dc8a93f1-1ee6-450d-90a3-8fa77f245b3b","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_x_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_x_bf)\nvalid_preds = logreg.predict(valid_x_bf)\nprint('Validation Xception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Xception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17e0f22ff8cea58ba198e5556c472b7dc6ed84a0","_cell_guid":"3fe2d11d-4b69-4bc1-b004-6fc84f56dd0e"},"cell_type":"markdown","source":"\nMuch better! 98% accuracy 0.07 LogLoss.****"},{"metadata":{"_uuid":"e17c08d790ef172e13900624df42d9ea9a43cc69","_cell_guid":"8ca30754-3b63-4c2a-a7ed-a111a4e0b837"},"cell_type":"markdown","source":"# Extract Inception bottleneck features"},{"metadata":{"_uuid":"5244f45ca3093adc9034a4115d309349f8b7bc1e","_cell_guid":"1c608857-0f1a-4132-b004-a1a3eda59d60","trusted":true},"cell_type":"code","source":"Xtr = x_train[train_idx]\nXv = x_train[valid_idx]\nprint((Xtr.shape, Xv.shape, ytr.shape, yv.shape))\ninception_bottleneck = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling=POOLING)\ntrain_i_bf = inception_bottleneck.predict(Xtr, batch_size=32, verbose=1)\nvalid_i_bf = inception_bottleneck.predict(Xv, batch_size=32, verbose=1)\nprint('InceptionV3 train bottleneck features shape: {} size: {:,}'.format(train_i_bf.shape, train_i_bf.size))\nprint('InceptionV3 valid bottleneck features shape: {} size: {:,}'.format(valid_i_bf.shape, valid_i_bf.size))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86550ce360964aa21f7cc70e1761132241e0cf55","_cell_guid":"d7930ef0-2d75-409c-88c7-b150f384a93e"},"cell_type":"markdown","source":"# LogReg on Inception bottleneck features"},{"metadata":{"_uuid":"e36b51ff6bba98246c6f380fecc33680deff88c5","_cell_guid":"675864f8-7bfe-47d6-960e-d4bd686cf31a","trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(train_i_bf, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(valid_i_bf)\nvalid_preds = logreg.predict(valid_i_bf)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8fe401e1ce9d41c617bd24045d4462a05f59eb88","_cell_guid":"655a9cfe-a2f2-4a8a-90d4-dac18cf72027","trusted":true},"cell_type":"code","source":"print('Validation Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea2d8acf56c3a2796c5923e3a75d3ac64d22bce7","_cell_guid":"b7b3f4f3-f134-4b3c-aaa0-9d7dd01b4b98"},"cell_type":"markdown","source":"# LogReg on all bottleneck features"},{"metadata":{"_uuid":"9356a162d8a39db89aab783d39c4012edd2a8ed5","_cell_guid":"1c858bb8-9d3a-4f30-8c07-e08726d471ac","trusted":true},"cell_type":"code","source":"X = np.hstack([train_x_bf, train_i_bf])\nV = np.hstack([valid_x_bf, valid_i_bf])\nprint('Full train bottleneck features shape: {} size: {:,}'.format(X.shape, X.size))\nprint('Full valid bottleneck features shape: {} size: {:,}'.format(V.shape, V.size))\nlogreg = LogisticRegression(multi_class='multinomial', solver='lbfgs', random_state=SEED)\nlogreg.fit(X, (ytr * range(NUM_CLASSES)).sum(axis=1))\nvalid_probs = logreg.predict_proba(V)\nvalid_preds = logreg.predict(V)\nprint('Validation Xception + Inception LogLoss {}'.format(log_loss(yv, valid_probs)))\nprint('Validation Xception + Inception Accuracy {}'.format(accuracy_score((yv * range(NUM_CLASSES)).sum(axis=1), valid_preds)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a8c185d2f7068a8d370aa239868d6a83b786960e","_cell_guid":"d6d917af-8c5a-4a06-ab3c-c7fb327e954e"},"cell_type":"markdown","source":"Training this model on the full dataset would give 0.3 LogLoss on LB."},{"metadata":{"_uuid":"646d482fb5c7b2a792abfbb6c05dd72a94f353a7","_cell_guid":"28ac6c7c-99c0-4afa-b396-58a5524e6ebc"},"cell_type":"markdown","source":"# Check errors\nWe still have a few misclassification errors."},{"metadata":{"_uuid":"17ae76c56f08f8602ce0456c9b14f33581da76d5","_cell_guid":"ff6da045-b279-4d49-a581-41fe6525b797","trusted":true},"cell_type":"code","source":"valid_breeds = (yv * range(NUM_CLASSES)).sum(axis=1)\nerror_idx = (valid_breeds != valid_preds)\nfor img_id, breed, pred in zip(labels.loc[valid_idx, 'id'].values[error_idx],\n                                [selected_breed_list[int(b)] for b in valid_preds[error_idx]],\n                                [selected_breed_list[int(b)] for b in valid_breeds[error_idx]]):\n    fig, ax = plt.subplots(figsize=(5,5))\n    img = read_img(img_id, 'train', (299, 299))\n    ax.imshow(img / 255.)\n    ax.text(10, 250, 'Prediction: %s' % pred, color='w', backgroundcolor='r', alpha=0.8)\n    ax.text(10, 270, 'LABEL: %s' % breed, color='k', backgroundcolor='g', alpha=0.8)\n    ax.axis('off')\n    plt.show()                                                    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d888c16718067eabe7540f4d2573ffdda24887c9","_cell_guid":"467d7924-aff9-4947-915c-77bc0a21ae13","trusted":true},"cell_type":"code","source":"end = dt.datetime.now()\nprint('Total time {} s.'.format((end - start).seconds))\nprint('We almost used the one hour time limit.')","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"mimetype":"text/x-python","pygments_lexer":"ipython3","name":"python","nbconvert_exporter":"python","codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","version":"3.6.3"},"kernelspec":{"language":"python","name":"python3","display_name":"Python 3"}},"nbformat":4,"nbformat_minor":1}