{"metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}}, "nbformat": 4, "cells": [{"metadata": {"_uuid": "bf55b7981ee043ec907d6b0634c56e55a1c4369d", "_cell_guid": "0365369d-e443-4b35-aa90-7b3762d2e83b"}, "execution_count": null, "cell_type": "code", "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "outputs": []}, {"metadata": {"_uuid": "61907af48dec6938d79a2c3043b7e78f21bc07a5", "_cell_guid": "baf85875-fb06-4f9c-8648-482d08764d4c"}, "execution_count": null, "cell_type": "code", "source": ["from keras.models import *\n", "from keras.layers import *\n", "from keras.applications import *\n", "from keras.preprocessing.image import *\n", "\n", "import h5py"], "outputs": []}, {"metadata": {"_uuid": "165f3f0de141f4b7d65ce8b9cef967379faf2552", "_cell_guid": "faefd94a-9adb-4ff7-b5ff-b60f88993a8d", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["from keras import applications\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras.preprocessing import image\n", "from keras.models import Sequential, load_model, Model\n", "from keras.layers import Activation, Dropout, Flatten, Dense, GlobalAveragePooling2D\n", "from keras.optimizers import SGD\n", "from keras.callbacks import EarlyStopping\n", "from keras.applications.vgg16 import VGG16\n", "from keras.applications.vgg16 import preprocess_input, decode_predictions\n", "from sklearn.utils import shuffle\n", "import os\n", "import h5py\n", "import numpy as np\n", "from keras.preprocessing.image import ImageDataGenerator\n", "from keras.models import Sequential\n", "from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D\n", "from keras.layers import Activation, Dropout, Flatten, Dense\n", "\n"], "outputs": []}, {"metadata": {"_uuid": "99c8f4131a4450e61535c665f1665433260eaa48", "_cell_guid": "d1b7f08d-a3d0-46d3-bab1-6bc0e8401ff2"}, "execution_count": null, "cell_type": "code", "source": ["data_root_dir = '../input/dogs-vs-cats-redux-kernels-edition/'\n", "keras_models_dir = '../input/keras-models/'\n", "kaggle_working = '/kaggle/working/'\n", "#print(check_output([\"ls\", keras_models_dir]).decode(\"utf8\"))\n", "print(check_output([\"ls\", data_root_dir]).decode(\"utf8\"))\n", "#print(check_output([\"ls\", kaggle_working]).decode(\"utf8\"))"], "outputs": []}, {"metadata": {"_uuid": "47967bfb11696facf1e9fd7e65f8094d5cfb6b88", "_cell_guid": "aa86d32e-677a-46f7-9714-e06f17ade06b", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["img_width, img_height = 224, 224\n", "\n", "def load_split_weights(model, model_path_pattern='model_%d.h5', memb_size=102400000):  \n", "    model_f = h5py.File(model_path_pattern, \"r\", driver=\"family\", memb_size=memb_size)\n", "    topology.load_weights_from_hdf5_group_by_name(model_f, model.layers)\n", "    return model"], "outputs": []}, {"metadata": {"_uuid": "6b1b83887c591ed7cb58642410dbb3f71945f067", "_cell_guid": "8ad402bb-1184-4e20-ad14-636fb5bb736d", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": [" nb_classes = 1\n", "\n", "def get_model():\n", "    vgg16 = applications.VGG16(include_top=False, weights=None) #input_shape = (3, img_width, img_height)\n", "    model_path_pattern = keras_models_dir + \"vgg16_weights_tf_dim_ordering_tf_kernels_%d.h5\" \n", "    vgg16 = load_split_weights(vgg16, model_path_pattern = model_path_pattern)\n", "\n", "   \n", "    # set the first 25 layers (up to the last conv block) to non-trainable (weights will not be updated)\n", "    for layer in vgg16.layers[:25]:\n", "        layer.trainable = False\n", "\n", "    x = vgg16.get_layer('block5_conv3').output\n", "    x = GlobalAveragePooling2D()(x)\n", "    x = Dense(256, activation='relu')(x)\n", "    x = Dropout(0.2)(x)\n", "    x = Dense(nb_classes, activation='sigmoid')(x)\n", "\n", "    model = Model(inputs=vgg16.input, outputs=x)\n", "\n", "    model.summary()\n", "    return model"], "outputs": []}, {"metadata": {"_uuid": "ddd2c1484456172ef01c79fe3da3753f10c294f5", "_cell_guid": "749a596d-b3e6-4ca8-8de3-b705f6ffa1ca", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["def get_model_include_top():\n", "    vgg16 = applications.VGG16(include_top=True, weights=None) #input_shape = (3, img_width, img_height)\n", "    model_path_pattern = keras_models_dir + \"vgg16_weights_tf_dim_ordering_tf_kernels_%d.h5\" \n", "    vgg16 = load_split_weights(vgg16, model_path_pattern = model_path_pattern)\n", "\n", "    #vgg16.summary()\n", "    \n", "    x = vgg16.get_layer('fc2').output\n", "    x = Dense(1, activation='sigmoid')(x)\n", "\n", "    model = Model(inputs=vgg16.input, outputs=x)\n", "    model.summary()\n", "    return model"], "outputs": []}, {"metadata": {"_uuid": "8a1aba1db8cdb50542f170ddc144180325702a88", "_cell_guid": "ab92a75b-9aa1-4587-a3d5-153b18f6df84"}, "execution_count": null, "cell_type": "code", "source": ["model = get_model_include_top()\n", "\n", "# set the first 25 layers (up to the last conv block) to non-trainable (weights will not be updated)\n", "\n", "for layer in model.layers:\n", "    layer.trainable = False\n", "    \n", "model.layers[-1].trainable=True\n", "model.layers[-2].trainable=True\n", "model.layers[-3].trainable=True"], "outputs": []}, {"metadata": {"_uuid": "c391d08ae08e43600e8e9efb45d93d965c419e02", "_cell_guid": "de440895-cc3b-4f2e-aa5e-e2987ced7fea"}, "execution_count": null, "cell_type": "code", "source": ["# set the first 25 layers (up to the last conv block) to non-trainable (weights will not be updated)\n", "import pandas as pd\n", "\n", "df = pd.DataFrame(([layer.name, layer.trainable] for layer in model.layers), columns=['layer', 'trainable'])\n", "df.style.applymap(lambda trainable: f'background-color: {\"yellow\" if trainable else \"red\"}', subset=['trainable'])"], "outputs": []}, {"metadata": {"_uuid": "3496065ce98084dfd336282c35a98e1a44e12c76", "_cell_guid": "9f93bc9a-527a-4708-9223-237007fd72b2", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])"], "outputs": []}, {"metadata": {"_uuid": "d47ca7e3cbd04fb4a5754d3420759cd67e56f542", "_cell_guid": "61365efc-4604-4ea1-9ca7-816cd8aef54b", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["def augment(src, choice):\n", "    if choice == 0:\n", "        # Rotate 90\n", "        src = np.rot90(src, 1)\n", "    if choice == 1:\n", "        # flip vertically\n", "        src = np.flipud(src)\n", "    if choice == 2:\n", "        # Rotate 180\n", "        src = np.rot90(src, 2)\n", "    if choice == 3:\n", "        # flip horizontally\n", "        src = np.fliplr(src)\n", "    if choice == 4:\n", "        # Rotate 90 counter-clockwise\n", "        src = np.rot90(src, 3)\n", "    if choice == 5:\n", "        # Rotate 180 and flip horizontally\n", "        src = np.rot90(src, 2)\n", "        src = np.fliplr(src)\n", "    return src"], "outputs": []}, {"metadata": {"_uuid": "196cab0182c846cf55473344caf5684be7a55b38", "_cell_guid": "8cfeb9e5-e717-456f-93c7-95e8cc10e85a"}, "execution_count": null, "cell_type": "code", "source": ["import glob\n", "from sklearn.model_selection import train_test_split\n", "from numpy import random\n", "import seaborn\n", "\n", "train_dogs = glob.glob(data_root_dir + \"train/dog.*\")\n", "train_cats = glob.glob(data_root_dir + \"train/cat.*\")\n", "#print (train_cats[:1])\n", "\n", "sample = 500\n", "# slice datasets for memory efficiency on Kaggle Kernels, delete if using full dataset\n", "images = train_dogs[:sample] + train_cats[:sample]\n", "random.shuffle(images)\n", "\n", "#print(images[:2])\n", "labels = []\n", "for i in images:\n", "    #print(i)\n", "    if \"dog.\" in i:\n", "        labels.append(1)\n", "    else:\n", "        labels.append(0)\n", "        \n", "def process_img(i):\n", "    img = load_img(dogs[i])  # this is a PIL image\n", "    x = img_to_array(img)  # this is a Numpy array with shape (3, 150, 150)\n", "    #x = x.reshape((1,) + x.shape)  # this is a Numpy array with shape (1, 3, 150, 150)\n", "    return x;\n", "\n", "train_images, validation_images = train_test_split(images, test_size=0.4)\n", "\n", "#print(\"Train shape: {}\".format(train.shape))\n", "#print(\"Test shape: {}\".format(test.shape))\n", "#print(\"Validation shape: {}\".format(valid.shape))\n", "#print(len(train_images))\n", "print(train_images[:1])\n", "print(validation_images[:1])\n", "\n", "seaborn.countplot(labels)\n", "seaborn.plt.title('Cats and Dogs')"], "outputs": []}, {"metadata": {"_uuid": "5c3aee421d1afef89211ae025b0754eaabde30ef", "_cell_guid": "58e19e32-f0b5-44f0-aa47-1d77a1e2997c"}, "cell_type": "markdown", "source": ["http://sujitpal.blogspot.com.au/2017/02/using-keras-imagedatagenerator-with.html"]}, {"metadata": {"_uuid": "963f9542497fc13936698d87552f63a8764acfc0", "_cell_guid": "1e155817-ed90-4524-b904-46ee01bb1a90", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["import matplotlib.pyplot as plt\n", "from scipy.misc import imresize\n", "from PIL import Image\n", "\n", "image_cache = {}\n", "\n", "def cached_imread(image_path):\n", "    if not image_path in image_cache:\n", "        img = Image.open(image_path) \n", "        if img.size != (img_width, img_height):\n", "            img = img.resize((img_width, img_height))\n", "    \n", "        x = image.img_to_array(img)\n", "        image_cache[image_path] = x\n", "    return image_cache[image_path]\n", "\n", "def preprocess_input_vgg(x):\n", "    #from keras.applications.resnet50 import preprocess_input\n", "    from keras.applications.vgg16 import preprocess_input\n", "    X = np.expand_dims(x, axis=0)\n", "    X = preprocess_input(X)\n", "    return X[0]\n", "\n", "def preprocess_images(image_names, seed, datagen, is_arg_enabled=True):\n", "    #print (image_names)\n", "    np.random.seed(seed)\n", "    X = np.zeros((len(image_names), img_width, img_height, 3))\n", "    for i, image_name in enumerate(image_names):\n", "        #print (image_name)\n", "        image = cached_imread(image_name)\n", "        if is_arg_enabled:\n", "            X[i] = datagen.random_transform(image)\n", "    return X\n", "\n", "def image_triple_generator(train_images, batch_size,is_arg_enabled=True):\n", "    datagen_args = dict(preprocessing_function=preprocess_input_vgg,\n", "                        rotation_range=10,\n", "                        width_shift_range=0.2,\n", "                        height_shift_range=0.2,\n", "                        shear_range=0.2,\n", "                        zoom_range=0.2,\n", "                        horizontal_flip=True)\n", "    datagen = ImageDataGenerator(**datagen_args)\n", "    \n", "    while True:\n", "        # loop once per epoch\n", "        num_recs = len(train_images)\n", "        #print(num_recs)\n", "        indices = np.random.permutation(np.arange(num_recs))\n", "        num_batches = num_recs // batch_size\n", "        for bid in range(num_batches):\n", "            # loop once per batch\n", "            batch_indices = indices[bid * batch_size : (bid + 1) * batch_size]\n", "            #print(batch_indices)\n", "            batch = [train_images[i] for i in batch_indices]\n", "            #print(batch)\n", "            # make sure image data generators generate same transformations\n", "            seed = np.random.randint(low=0, high=1000, size=1)[0]\n", "            batch_label = []\n", "            batch_img = preprocess_images(batch, seed, datagen,is_arg_enabled=True)\n", "            for i in batch:\n", "                if \"dog.\" in i:\n", "                    batch_label.append(1)\n", "                else:\n", "                    batch_label.append(0)\n", "            \n", "            batch_labels = to_categorical(batch_labels,nb_classes)\n", "                    \n", "            yield batch_img, batch_label\n", "\n", "batch_size = 2\n", "batches = image_triple_generator(train_images, batch_size)\n", "val_batches = image_triple_generator(validation_images, batch_size, is_arg_enabled=False)"], "outputs": []}, {"metadata": {"_uuid": "3433d8f7f85a98907e426b9bbc35ec80de067f61", "_cell_guid": "050e48c7-85a8-435a-b33d-f38ecbb3d710"}, "execution_count": null, "cell_type": "code", "source": ["trn_classes = len(train_images)\n", "val_classes = len(validation_images)\n", "steps_per_epoch=int(np.ceil(trn_classes/batch_size))+1\n", "validation_steps=int(np.ceil(val_classes/batch_size))+1   \n", "epochs=15\n", "\n", "print (\"epochs:\" + str(epochs))\n", "print (\"batch_size:\" + str(batch_size))\n", "print (\"trn_classes:\" + str(trn_classes))\n", "print (\"val_classes:\" + str(val_classes))\n", "print (\"steps_per_epoch:\" + str(steps_per_epoch))\n", "print (\"validation_steps:\" + str(validation_steps))"], "outputs": []}, {"metadata": {"_uuid": "5755d4b6ff4c5252f8bb12526ae6f99c9ed7f85d", "_cell_guid": "79347ed0-d919-40ed-ac56-e3077d9cf948", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["from keras.preprocessing import image\n", "from keras.callbacks import ModelCheckpoint, EarlyStopping\n", "from keras.callbacks import LearningRateScheduler\n", "import os.path, os\n", "\n", "fine_weights_path = kaggle_working + 'tune_weights.h5'\n", "\n", "\n", "if os.path.isfile(fine_weights_path) :\n", "    print (\"rm fine_weights_path:\" + fine_weights_path)\n", "    #model.load_weights(fine_weights_path)\n", "    os.remove(fine_weights_path)\n", "    \n", "def step_decay(epoch):\n", "    if epoch >= 0 and epoch < 2:\n", "        lrate = 0.001 #Default Adam lr=0.001\n", "    elif epoch >= 2 and epoch < 10:\n", "        lrate = 0.0001\n", "    elif epoch >= 5 and epoch < 10:\n", "        lrate = 0.00001\n", "    elif epoch >= 15 and epoch < 20:\n", "        lrate = 0.000001\n", "    else:\n", "        lrate = 0.000001\n", "    \n", "    print (str(epoch) + \" learning rate:%.6f\" % lrate)\n", "    return lrate\n", "\n", "reduce_lr = LearningRateScheduler(step_decay)\n", "\n", "#Removed due to err 'no space left on device'\n", "#ModelCheckpoint(fine_weights_path, monitor='val_acc', verbose=1, save_best_only=True),\n", "callbacks_list = [\n", "    EarlyStopping(monitor='val_acc', patience=5, verbose=1),reduce_lr]"], "outputs": []}, {"metadata": {"_uuid": "2e4b27241086f8d0f870c59d9d368466331bd739", "_cell_guid": "53771d87-135b-4b9f-8765-4263851e4201"}, "execution_count": null, "cell_type": "code", "source": ["history = model.fit_generator(batches, \n", "                    steps_per_epoch=steps_per_epoch, \n", "                    epochs=epochs, \n", "                    validation_data=val_batches, \n", "                    validation_steps=validation_steps,\n", "                    callbacks=callbacks_list,          \n", "                    verbose=1)"], "outputs": []}, {"metadata": {"_uuid": "eefdc9a94c02787c94288327cc9bad9128b3aa7a", "_cell_guid": "047bcf6c-ba04-4d0c-b3c4-9cd0f1dfb28d", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": [" print(\"Training accuracy: %.2f%% / Validation accuracy: %.2f%%\" % (100*history.history['acc'][-1], 100*history.history['val_acc'][-1]))"], "outputs": []}, {"metadata": {"_uuid": "6d4c94cce89fc37a38bc009588dad24e3cb17003", "_cell_guid": "0b0dec9d-b993-4293-9fd1-ff1251b16382", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["#model.save_weights(fine_weights_path)\n", "#os.remove(fine_weights_path)"], "outputs": []}, {"metadata": {"_uuid": "1e26d738563ddc0db8a7d5df58fe102b0c3e6ba1", "_cell_guid": "f6735475-93af-457a-a468-0c742f492223", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["import matplotlib.pyplot as plt\n", "\n", "# list all data in history\n", "print(history.history.keys())\n", "\n", "plt.plot(history.history['val_acc'])\n", "plt.plot(history.history['acc'])\n", "plt.title('model accuracy')\n", "plt.ylabel('accuracy')\n", "plt.xlabel('epoch')\n", "plt.legend(['train', 'validation'], loc='upper left')\n", "plt.show()\n", "# summarize history for loss\n", "plt.plot(history.history['loss'])\n", "plt.plot(history.history['val_loss'])\n", "plt.title('model loss')\n", "plt.ylabel('loss')\n", "plt.xlabel('epoch')\n", "plt.legend(['train', 'validation'], loc='upper left')\n", "plt.show()"], "outputs": []}, {"metadata": {"_uuid": "034996bafb90357ceb79192b38d83586c1de53b8", "_cell_guid": "84bea76e-0505-4d8a-8c66-0bc65a9936ef", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["#predictions = model.predict(X_test, verbose=0)"], "outputs": []}, {"metadata": {"_uuid": "ad6b02e0d529ce7b4529f3cb9c9f7219bd8a79b4", "_cell_guid": "a2505384-b6c6-461c-8435-6367f2f2e35e", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["#y_pred = model2.predict(X_test, verbose=1)\n", "#y_pred = y_pred.clip(min=0.005, max=0.995)"], "outputs": []}, {"metadata": {"_uuid": "b6474a4d0bee0472118ee9fe4c37c8eb66988a22", "_cell_guid": "c0b58021-17c2-4645-81c6-3e9c18fda6c4", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": ["#from IPython.display import FileLink, FileLinks\n", "\n", "#FileLink('submission.csv')"], "outputs": []}, {"metadata": {"_uuid": "1a11a7cb80648bdf16340276d6a49aab4f40752f", "_cell_guid": "dcdb2986-f6f7-4318-bd25-c6f0319f6e1d", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": [], "outputs": []}, {"metadata": {"_uuid": "83d82ca8b646de1f0a86097be92f89cc8a8b941c", "_cell_guid": "8023623d-591a-4f2a-9863-f5beca05a6f9", "collapsed": true}, "execution_count": null, "cell_type": "code", "source": [], "outputs": []}], "nbformat_minor": 1}