{"cells":[{"metadata":{"_cell_guid":"f23174c2-fdc2-4c88-bb78-b6bf6f7fc859","_uuid":"382566de006529d3f5b7e7d144ae4b948141f12e","trusted":true},"cell_type":"code","source":"import tensorflow as tf","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"c4a3bdb1-2d9e-4385-857d-80614a7a19d8","collapsed":true,"_uuid":"91f1513014de5b47f85df919692df2a756fbaca2","trusted":true},"cell_type":"code","source":"# Common imports\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport os\n\nnp.random.seed(42)\n\n# to make this notebook's output stable across runs\ndef reset_graph(seed=42):\n    tf.reset_default_graph()\n    tf.set_random_seed(seed)\n    np.random.seed(seed)\n\n# To plot pretty figures\n%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\nplt.rcParams['axes.labelsize'] = 14\nplt.rcParams['xtick.labelsize'] = 12\nplt.rcParams['ytick.labelsize'] = 12","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"a5bd3e24-179c-4fe5-84dc-88537abb35ae","collapsed":true,"_uuid":"f0537ca9de8f69647edab39fd3d83e59902bd7ee","trusted":true},"cell_type":"code","source":"from sklearn.datasets import load_sample_image\n\n#To plot images\n\ndef plot_color_image(image):\n    plt.imshow(image.astype(np.uint8),interpolation=\"nearest\")\n    plt.axis(\"off\")","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"9d049aab-8cf7-46ad-addd-e48509368aa4","collapsed":true,"_uuid":"13f76e40cc1cf275dc6960fea3786795babc35d0","trusted":true},"cell_type":"code","source":"import matplotlib.image as mpimg","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"87e1a981-03ac-4ca4-87c8-da950b9d6e9f","_uuid":"2e2713c4f5bbbbc2b446c26a3e4567b13fd03bf7"},"cell_type":"markdown","source":"Here we have a look the images that we have"},{"metadata":{"scrolled":true,"_cell_guid":"76e2f412-5933-4e07-b97b-1d77a95e254f","_uuid":"fa487a1b03427ca2350c843b66f43faed9f8ca80","trusted":true},"cell_type":"code","source":"test_image = mpimg.imread(os.path.join(\"../input/train\", \"cat.0.jpg\"))\nplot_color_image(test_image)","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"ed024254-3999-43ea-8f71-2d89a3f10a5b","_uuid":"ebdb4eeebaecf6d91c5ac46e415d83ba9d76efac"},"cell_type":"markdown","source":"This is the path where we have the data. In order for this to be reproducible, this\npath has to be changed where the data is. This notebook should be in the same folder as data \n(in this case, CatsDogs). We see that the animal classes are cats and dogs"},{"metadata":{"_cell_guid":"77279c84-5d9d-4282-81c9-bca863f9d48a","collapsed":true,"_uuid":"aa3970e05a89f44004019fd1cc6d3c1195da16a8","trusted":true},"cell_type":"code","source":"animals_root_path = '../input'\n\nanimals_classes = ['cat', 'dog']","execution_count":21,"outputs":[]},{"metadata":{"_cell_guid":"87cb31e0-048e-434e-b655-94397b7e0058","_uuid":"601ecd09555ac9bbc35054e2fec313037578b7ef"},"cell_type":"markdown","source":"Image paths is a dictionary that has as keys cats and dogs (strings) and as values a list of the paths of the images that have either cats or dogs"},{"metadata":{"_cell_guid":"19377f6f-23ec-413e-ac23-bf58406ebd8d","collapsed":true,"_uuid":"4cae71be0090c0fa0194b798f3146d584d189a25","trusted":true},"cell_type":"code","source":"from collections import defaultdict\nimport re\n\nimage_paths = defaultdict(list)\n\ntrain_path = os.path.join(animals_root_path, 'train')\n\nfor animal in animals_classes:\n    for filepath in os.listdir(train_path):\n        if filepath.startswith(animal):\n            image_paths[animal].append(os.path.join(train_path, filepath))","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"d09fe1f2-5037-4edc-a550-4b73be38df58","_uuid":"3815a35a34731fb10d964ec215aa2272bfbde5c4","trusted":true},"cell_type":"code","source":"image_paths['cat'][0]","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"6a9dde55-8700-4a4f-bf38-45edefd9114d","_uuid":"ae729e155b658368c8bcc35dd3efbd0686f31d22"},"cell_type":"markdown","source":"Here we sort the paths, for no reason"},{"metadata":{"_cell_guid":"ffc4f99e-f4a6-4e9f-9d29-0cc23602b505","collapsed":true,"_uuid":"4d8ab11812810bb53a29e20d421bf1164216328a","trusted":true},"cell_type":"code","source":"for paths in image_paths.values():\n    paths.sort()    ","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"2f55cc34-c0b1-4781-a055-dcabd898b5a0","_uuid":"3342053988d38fb8b8b0f9486b3b9bc17470e30b"},"cell_type":"markdown","source":"We can see some of the pictures that correspond to cats and dogs"},{"metadata":{"scrolled":true,"_cell_guid":"e27bd3e9-d2b5-40bd-affc-cddfc8977110","_uuid":"b9c899e4e874038cefb5bd57b91e59142e460311","trusted":true},"cell_type":"code","source":"import matplotlib.image as mpimg\n\nn_examples_per_class = 6\nchannels = 3\n\nfor animal in animals_classes:\n    print(\"Class:\", animal)\n    plt.figure(figsize=(10,5))\n    for index, example_image_path in enumerate(image_paths[animal][:n_examples_per_class]):\n        example_image = mpimg.imread(example_image_path)[:, :, :channels]\n        plt.subplot(100 + n_examples_per_class * 10 + index + 1)\n        plt.title(\"{}x{}\".format(example_image.shape[1], example_image.shape[0]))\n        plt.imshow(example_image)\n        plt.axis(\"off\")\n    plt.show()","execution_count":25,"outputs":[]},{"metadata":{"_cell_guid":"7fa25118-09f4-4386-9dfb-92896590adea","_uuid":"d96690d363c6e565101788246d2ea04409be4ebb"},"cell_type":"markdown","source":"This is a function to reshape the image. We use some data augmentation as well"},{"metadata":{"_cell_guid":"1064b373-e140-43e2-b357-f55641d8eeca","collapsed":true,"_uuid":"8a535f6e268918f7da4369ca5133b5c7e2298808","trusted":true},"cell_type":"code","source":"from scipy.misc import imresize\nfrom skimage.transform import resize\n\ndef prepare_image(image, target_width = 80, target_height = 80, max_zoom = 0.2):\n    \"\"\"Zooms and crops the image randomly for data augmentation.\"\"\"\n\n    # First, let's find the largest bounding box with the target size ratio that fits within the image\n    height = image.shape[0]\n    width = image.shape[1]\n    image_ratio = width / height\n    target_image_ratio = target_width / target_height\n    crop_vertically = image_ratio < target_image_ratio\n    crop_width = width if crop_vertically else int(height * target_image_ratio)\n    crop_height = int(width / target_image_ratio) if crop_vertically else height\n        \n    # Now let's shrink this bounding box by a random factor (dividing the dimensions by a random number\n    # between 1.0 and 1.0 + `max_zoom`.\n    resize_factor = np.random.rand() * max_zoom + 1.0\n    crop_width = int(crop_width / resize_factor)\n    crop_height = int(crop_height / resize_factor)\n    \n    # Next, we can select a random location on the image for this bounding box.\n    x0 = np.random.randint(0, width - crop_width)\n    y0 = np.random.randint(0, height - crop_height)\n    x1 = x0 + crop_width\n    y1 = y0 + crop_height\n    \n    # Let's crop the image using the random bounding box we built.\n    image = image[y0:y1, x0:x1]\n\n    # Let's also flip the image horizontally with 50% probability:\n    if np.random.rand() < 0.5:\n        image = np.fliplr(image)\n\n    # Now, let's resize the image to the target dimensions.\n    image = resize(image, (target_width, target_height), mode = 'constant')\n    # Finally, let's ensure that the colors are represented as\n    # 32-bit floats ranging from 0.0 to 1.0 (for now):\n    return image.astype(np.float32) #/ 255","execution_count":26,"outputs":[]},{"metadata":{"_cell_guid":"45254608-6d66-4d9a-b6ff-fe8481287032","collapsed":true,"_uuid":"29c3bd0ccffa9c505363c264a6b95ea5c89041a2"},"cell_type":"markdown","source":"Here we can see an image of a dog, and later how the transformations via the previous function change the image"},{"metadata":{"scrolled":true,"_cell_guid":"c8013eaa-c677-4905-8ac4-01957d104a5f","_uuid":"554db4f5551fad62b50cf9187e6b0f181c66bae2","trusted":true},"cell_type":"code","source":"plt.figure(figsize=(6, 8))\nplt.imshow(example_image)\nplt.title(\"{}x{}\".format(example_image.shape[1], example_image.shape[0]))\nplt.axis(\"off\")\nplt.show()","execution_count":27,"outputs":[]},{"metadata":{"_cell_guid":"c04d6267-5a17-482b-bc75-aca79d5922a9","_uuid":"a2de6b68ae5830eb977e72f105706cea3df42da0"},"cell_type":"markdown","source":"Let's see the transformations"},{"metadata":{"_cell_guid":"39c00c2d-32d2-468b-bf4f-4e8576edc15b","_uuid":"143eaa65021f26fea7d5946bd90041292e9c5c48","trusted":true},"cell_type":"code","source":"rows, cols = 2, 3\n\nplt.figure(figsize=(14, 8))\nfor row in range(rows):\n    for col in range(cols):\n        prepared_image = prepare_image(example_image)\n        plt.subplot(rows, cols, row * cols + col + 1)\n        plt.title(\"{}x{}\".format(prepared_image.shape[1], prepared_image.shape[0]))\n        plt.imshow(prepared_image)\n        plt.axis(\"off\")\nplt.show()","execution_count":28,"outputs":[]},{"metadata":{"_cell_guid":"3740f99b-b9f9-4cf7-b4f1-8779feb61108","_uuid":"2a5bf1739c27806eedcecce7aa2a0ed9e2461999"},"cell_type":"markdown","source":"Time to create the tensorflow graph. We will use convolution, padding, pooling, batchnorm and relus"},{"metadata":{"_cell_guid":"e02ac382-3f99-4e9a-8afc-77c6f5840d88","_uuid":"83e9e1a47c2fdb10b8e4aa2f69f4dd7acd2ecf97","trusted":true},"cell_type":"code","source":"height = 80\nwidth = 80\nchannels = 3\nn_inputs = height * width * channels\n\n\n\n\nconv1_fmaps = 32\nconv1_ksize = 3\nconv1_stride = 1\nconv1_pad = \"SAME\"\n\nconv2_fmaps = 64\nconv2_ksize = 3\nconv2_stride = 1\nconv2_pad = \"SAME\"\n\nconv3_fmaps = 128\nconv3_ksize = 3\nconv3_stride = 1\nconv3_pad = \"SAME\"\n\n\n\nn_fc1 = 32\n\n#learn_rate_value = 1e-3\n\nreset_graph()\n\nwith tf.name_scope(\"inputs\"):\n    X = tf.placeholder(tf.float32, shape=[None, height, width, channels], name=\"X\")\n    y = tf.placeholder(tf.float32, shape=[None], name=\"y\")\n    learning_rate = tf.placeholder(tf.float32, name='learning_rate')\n    iters_per_epoch = tf.placeholder(tf.int32, name = 'iters_per_epoch')\n\nglobal_step = tf.Variable(0)    \n\n\ndef convBlock(X, conv_fmaps, conv_ksize, conv_stride, \n              conv_pad, name_index):\n    conv = tf.layers.conv2d(X, filters=conv_fmaps, kernel_size=conv_ksize,\n                         strides=conv_stride, padding=conv_pad,\n                         activation=tf.nn.relu, name=\"conv\" + str(name_index))\n    bn = tf.layers.batch_normalization(conv, name = 'batchnorm' + str(name_index))\n    relu = tf.nn.relu(bn, name = 'relu' + str(name_index))\n    pool = tf.nn.max_pool(relu, ksize = [1, 2, 2, 1], strides = [1, 2, 2, 1],\n                           padding = 'VALID', name = 'pool' + str(name_index))\n    return pool\n    \nfirstBlock = convBlock(X, conv1_fmaps, conv1_ksize, conv1_stride, conv1_pad, '1')\nsecondBlock = convBlock(firstBlock, conv2_fmaps, conv2_ksize, conv2_stride, conv2_pad, '2')\npool3 = convBlock(secondBlock, conv3_fmaps, conv3_ksize, conv3_stride, conv3_pad, '3')\n\n\nwith tf.name_scope(\"fc1\"):\n    flat_inputs = tf.contrib.layers.flatten(pool3)\n    fc1 = tf.layers.dense(flat_inputs, n_fc1, activation=tf.nn.relu, name=\"fc1\")\n\nwith tf.name_scope(\"output\"):\n    logits = tf.layers.dense(fc1, units=1, name=\"output\")\n    logits = tf.reshape(logits, shape = [-1])\n    Y_proba = tf.nn.sigmoid(logits, name=\"Y_proba\")\n\nwith tf.name_scope(\"train\"):\n    learn_rate = tf.train.cosine_decay_restarts(learning_rate,\n                                                global_step,\n                                                iters_per_epoch,\n                                                alpha = learning_rate/10.,\n                                                name= 'LearningRate')\n    xentropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=y)\n    loss = tf.reduce_mean(xentropy)\n    training_op = tf.train.AdamOptimizer(learn_rate).minimize(loss,\n                                                              global_step=global_step)\n\n    optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate)\n    training_op = optimizer.minimize(loss)\n\n    \nwith tf.name_scope(\"learn_rate_finder\"):\n    xentropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=y)\n    loss = tf.reduce_mean(xentropy)\n    training_op_finder = tf.train.AdamOptimizer(learning_rate).minimize(loss,\n                                                       global_step=global_step)\n\n\nwith tf.name_scope(\"eval\"):\n    correct_prediction = tf.equal(tf.to_int32(Y_proba > 0.5),\n                                  tf.cast(y, tf.int32))\n    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n\nwith tf.name_scope(\"init_and_save\"):\n    init = tf.global_variables_initializer()\n    init_local = tf.local_variables_initializer()\n    saver = tf.train.Saver()","execution_count":29,"outputs":[]},{"metadata":{"_cell_guid":"548cace3-7c86-4534-8901-045b7a3b6848","_uuid":"549b75a06a7c05d9ade2a9506e2840131b0b41c5"},"cell_type":"markdown","source":"For convenience, we will create a dictionary that takes as keys the animal classes\nand as values its representation in binary"},{"metadata":{"_cell_guid":"084e4c56-e3b5-484e-9ede-ab85dd0971e0","_uuid":"38c6028308bc1e8c2afb8f77cfa2b7814ac76a19","trusted":true},"cell_type":"code","source":"animal_class_ids = {animal_class: index for index, animal_class in enumerate(animals_classes)}\nanimal_class_ids","execution_count":30,"outputs":[]},{"metadata":{"_cell_guid":"dce9d001-a79f-4f23-9c81-88b39981a1db","_uuid":"08b934a9a4b6c109475843494ff5a45f7e0db634"},"cell_type":"markdown","source":"Animals paths and classes is a list, and each element is a tupple that has the path of the image\nand its corresponding animal (in binary)"},{"metadata":{"_cell_guid":"17771f22-4948-4f1a-8fc9-f95793aafc3b","collapsed":true,"_uuid":"ee9b4ccc05ad1aa2bf9700ab77a2d219c890c51d","trusted":true},"cell_type":"code","source":"animals_paths_and_classes = []\nfor animal, paths in image_paths.items():\n    for path in paths:\n        animals_paths_and_classes.append((path, animal_class_ids[animal]))","execution_count":31,"outputs":[]},{"metadata":{"_cell_guid":"6c28b89e-6a81-4140-b9cd-712af4eb75bb","_uuid":"beb2a84b0e8aadc6aec98b211c323b23e5be80ce"},"cell_type":"markdown","source":"We create the validation and train file"},{"metadata":{"_cell_guid":"374f1af5-8498-4fc3-8a7c-f22d999a92e4","collapsed":true,"_uuid":"4434f73c2f282aa9856144bb36e903a15542a590","trusted":true},"cell_type":"code","source":"val_ratio = 0.01\ntrain_size = int(len(animals_paths_and_classes) * (1 - val_ratio))\n\nnp.random.shuffle(animals_paths_and_classes)\n\nanimals_paths_and_classes_train = animals_paths_and_classes[:train_size]\nanimals_paths_and_classes_val = animals_paths_and_classes[train_size:]","execution_count":32,"outputs":[]},{"metadata":{"_cell_guid":"e1513f0f-7457-4878-b0b1-5266a9f2d9f6","_uuid":"f51b1b55a15b1d07c23e2fc085ca3c6b1c3138c7"},"cell_type":"markdown","source":"The first function prepares a random batch and the second one prepares a batch given its indices\nin the animals_paths_and_classes list"},{"metadata":{"_cell_guid":"37098cd5-c2fd-4264-9bd1-e650ee90fba1","collapsed":true,"_uuid":"d5ee088a4ec0991dc2fc0401da8bb418eef4afc5","trusted":true},"cell_type":"code","source":"from random import sample\n\ndef prepare_batch(animals_paths_and_classes, batch_size):\n    batch_paths_and_classes = sample(animals_paths_and_classes, batch_size)\n    images = [mpimg.imread(path)[:, :, :channels] for path, labels in batch_paths_and_classes]\n    prepared_images = [prepare_image(image) for image in images]\n    X_batch = 2 * np.stack(prepared_images) - 1 # Inception expects colors ranging from -1 to 1\n    y_batch = np.array([labels for path, labels in batch_paths_and_classes], dtype=np.int32)\n    return X_batch, y_batch\n\ndef prepare_batch_indices(animals_paths_and_classes, indices):\n    batch_paths_and_classes = [animals_paths_and_classes[i] for i in indices]\n    images = [mpimg.imread(path)[:, :, :channels] for path, labels in batch_paths_and_classes]\n    prepared_images = [prepare_image(image) for image in images]\n    X_batch = 2 * np.stack(prepared_images) - 1 # Inception expects colors ranging from -1 to 1\n    y_batch = np.array([labels for path, labels in batch_paths_and_classes], dtype=np.int32)\n    return X_batch, y_batch","execution_count":33,"outputs":[]},{"metadata":{"_cell_guid":"14241025-795c-4d29-990c-f0930e7c105b","_uuid":"10b26886d656437f23b440a9e61622c9fb28b1dc"},"cell_type":"markdown","source":"We create the actual validation object"},{"metadata":{"_cell_guid":"ecb0a849-d8c4-44b2-bfb5-54bcce6d2c91","collapsed":true,"_uuid":"a58f7631ab996c1b4d6c863c1048883a7b776a24","trusted":true},"cell_type":"code","source":"def prepare_batch_total(animals_paths_and_classes):\n    batch_paths_and_classes = animals_paths_and_classes\n    images = [mpimg.imread(path)[:, :, :channels] for path, labels in batch_paths_and_classes]\n    prepared_images = [prepare_image(image) for image in images]\n    X_batch = 2 * np.stack(prepared_images) - 1 # Inception expects colors ranging from -1 to 1\n    y_batch = np.array([labels for path, labels in batch_paths_and_classes], dtype=np.int32)\n    return X_batch, y_batch\n\nX_val, y_val = prepare_batch_total(animals_paths_and_classes_val)","execution_count":34,"outputs":[]},{"metadata":{"_cell_guid":"57902b3f-94ad-48ef-a196-53ef9a06e87c","_uuid":"58dbc0cab2fe29e751e787dc4132518e79f35ff7"},"cell_type":"markdown","source":"This function is to do tta. First of all we will create many validation sets, and after that\nwe will compute its probabilities of cat or dog through the net in order to make predictions."},{"metadata":{"_cell_guid":"ef344f2f-2352-4261-a542-cbf1594ff60d","collapsed":true,"_uuid":"2ce6ee6039031a44a177b5183756d3c2a4c9dc9a","trusted":true},"cell_type":"code","source":"#This function creates an augmented validation set. It does so by creating a list.\n#Every element of the list is a different version of the validation data.\n\ndef create_test_data(animals_paths_and_classes_val, tta_len):\n    val_data_tta = []\n    for i in range(tta_len):\n        val_data_tta.append(prepare_batch_total(animals_paths_and_classes_val))\n    return val_data_tta","execution_count":35,"outputs":[]},{"metadata":{"_cell_guid":"9e06dfd6-1a30-450e-998d-d79e3b3dfce7","collapsed":true,"_uuid":"e49208c2bdfa383abddd8655fba666faab1108ed","trusted":true},"cell_type":"code","source":"val_data = create_test_data(animals_paths_and_classes_val, 5)","execution_count":36,"outputs":[]},{"metadata":{"_cell_guid":"ea9d9007-875d-4e22-b949-34cdeab93481","collapsed":true,"_uuid":"d2411627b88808eead73b4844cdb87e6bc52cef1","trusted":true},"cell_type":"code","source":"def computeAugmentedProbabilities(Y_proba, val_data):\n    probabilities = np.zeros((len(val_data[0][1]), len(val_data)))\n    i = 0\n    for X_val_tta, y_val_tta in val_data:\n        probabilities[:, i] = Y_proba.eval(feed_dict={X: X_val_tta, y: y_val_tta})\n        i += 1\n    return np.mean(probabilities, axis = 1)","execution_count":37,"outputs":[]},{"metadata":{"_cell_guid":"ef39c11d-2a8c-477c-86f2-116cf3596326","collapsed":true,"_uuid":"3c32836609b92e95d4b3cfebd24582a7a60d2091","trusted":true},"cell_type":"code","source":"def computeAugmentedAccuracy(Y_proba, val_data):\n    probs = computeAugmentedProbabilities(Y_proba, val_data)\n    labels = val_data[0][1]\n    predictions = probs > .5\n    return ((predictions == labels).sum())/len(labels)","execution_count":38,"outputs":[]},{"metadata":{"_cell_guid":"3091dbb4-5afc-4448-941b-6ce082e18bbc","_uuid":"0353401efb25015fd095878e633771413b966045"},"cell_type":"markdown","source":"This is a routine to evaluate the losses using diferent learning rates. Indices are\nthe indices of the batch that we are going to use to see the decrease of the loss function\nas a function of the learning rate. We do several steps (iters_per_learn_rate) in order to have\nan estimation of the decrease rate with that learning rate"},{"metadata":{"_cell_guid":"6708b1d1-bd51-4da2-a59e-9a028ee267fb","collapsed":true,"_uuid":"48565f354023a68c250de6cc704c7d1088d35848","trusted":true},"cell_type":"code","source":"def learning_rate_finder(learning_rates, indices, \n                         animals_paths_and_classes_train,\n                         iters_per_learn_rate):\n    losses = []\n    learn_rates_used = []\n    initial_losses = []\n    X_batch, y_batch = prepare_batch_indices(animals_paths_and_classes_train, indices)\n    for learn_rate in (learning_rates):\n        learn_rate_value = learn_rate\n        with tf.Session() as sess:\n            init.run(session = sess)\n            initial_loss = loss.eval(feed_dict={X: X_batch, y: y_batch}, session = sess)\n            initial_losses.append(initial_loss)\n            for iteration in (range(len(learning_rates))):\n                sess.run(training_op, feed_dict={X: X_batch, y: y_batch,\n                                                 learning_rate: learn_rate,\n                                                 })\n                losses.append(loss.eval(feed_dict={X: X_batch, y: y_batch}))\n                learn_rates_used.append(learn_rate)\n    return initial_losses, losses, learn_rates_used","execution_count":39,"outputs":[]},{"metadata":{"_cell_guid":"e77dc8e9-7b1c-4f4c-a509-bbfc830f0724","_uuid":"0339423bd40ab7a98dbcd572dc7e90e19b4fae9d"},"cell_type":"markdown","source":"These are the parameters that we are going to input to the previous function. "},{"metadata":{"_cell_guid":"979e344e-fd8b-429e-9a5b-44ba741b70e4","collapsed":true,"_uuid":"9c756372489ab2be7ae1e1eb2c1eec51e0f51a1c","trusted":true},"cell_type":"code","source":"indices = np.random.randint(0, len(animals_paths_and_classes_train), 128)\nlearning_rates = [5e-5, 1e-4, 5e-4, 1e-3, 5e-3, 1e-2, 5e-2, 1e-1, .5, 1.]\niters_per_learn_rate = 4","execution_count":40,"outputs":[]},{"metadata":{"_cell_guid":"969de961-5f99-473d-9c42-8c849dd53483","_uuid":"30217c2d14d3a7281d779359df7067232e5806de"},"cell_type":"markdown","source":"Here we run iterations using different learning rates:"},{"metadata":{"scrolled":true,"_cell_guid":"ae2d4467-1357-4a20-902b-94ee73ceade7","collapsed":true,"_uuid":"9f817a49803d6a316c7287edd25cc6a0d9b41076","trusted":true},"cell_type":"code","source":"initial_losses, losses, learn_rates = learning_rate_finder(learning_rates, indices, \n                                                           animals_paths_and_classes_train,\n                                                           iters_per_learn_rate)","execution_count":41,"outputs":[]},{"metadata":{"_cell_guid":"5b5e5e6e-8e57-4ddf-8fcf-37503b9321ae","_uuid":"dbffed69f2326b3fbf3862205bd0dbd2c0e54061"},"cell_type":"markdown","source":"As we can see, as we are always restarting and using the same batch, the initial loss is always the same"},{"metadata":{"_cell_guid":"973ef3f4-93f4-49bb-9785-b1cf857191d8","_uuid":"df65a3d0e2ce5f2ace527bc38c8bc4dc8349f389","trusted":true},"cell_type":"code","source":"initial_losses","execution_count":42,"outputs":[]},{"metadata":{"_cell_guid":"89106de2-9a1a-4423-b1fd-9609cd77b5a9","_uuid":"a26ca18c3deec99e3a05eb65241cf862426b744d"},"cell_type":"markdown","source":"We see which is the average improvement as a function of the learning rate. We take the learning rate with the largest improvement"},{"metadata":{"_cell_guid":"f215df63-f2c1-4cc4-af40-b562d534e498","_uuid":"69454986e118180ddf8dcb5c8a2b4e23e3f56695","trusted":true},"cell_type":"code","source":"lr_finder = pd.DataFrame({'Improvement': losses - initial_losses[0], 'Learning rate': learn_rates})\nlr_finder.groupby(by = 'Learning rate').mean()","execution_count":43,"outputs":[]},{"metadata":{"_cell_guid":"7eb4dce0-4d25-4a29-807c-f7d77f3cf6c1","collapsed":true,"_uuid":"001e3ab3da244d4ba170a8701b0e61e2d304e36b","trusted":true},"cell_type":"code","source":"means = lr_finder.groupby(by = 'Learning rate').mean()\nsorted_means = means.sort_values('Improvement')\nlearn_rate_value = sorted_means.index[0]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_cell_guid":"1251d131-e1fe-4d1f-bb31-933aff233ae0","_uuid":"845c876f554d5e67b59e3220d685941044d63d85","trusted":true},"cell_type":"code","source":"n_epochs = 15\nbatch_size = 128\n\n\ntrain_set_size = len(animals_paths_and_classes_train)\n\nn_iterations_per_epoch = int(len(animals_paths_and_classes_train)/batch_size)\n\n\nwith tf.Session() as sess:\n    init.run()\n    for epoch in range(0, n_epochs): # This will not be reall epochs, due to time constraints\n        for iteration in (range(0, n_iterations_per_epoch)):\n            indices = np.random.randint(0, train_set_size, size = batch_size)\n            X_batch, y_batch = prepare_batch_indices(animals_paths_and_classes_train,\n                                                     indices)\n            sess.run(training_op, feed_dict={X: X_batch, y: y_batch,\n                                             learning_rate: learn_rate_value,\n                                             iters_per_epoch: n_iterations_per_epoch})\n            #if iteration%20 == 0:\n        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})\n        acc_val = computeAugmentedAccuracy(Y_proba, val_data)\n        print('Epoch:', epoch, \"Train accuracy:\", acc_train, \"Val accuracy:\", acc_val)\n    save_path = saver.save(sess, \"./cats_dogs_model_v1\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"08b1d9b8-df8f-4590-9d58-c1793a4a49a9","collapsed":true,"_uuid":"c47d38679163ef20f0cffeedd6624f3784ebe7fa","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}