# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load in import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the "../input/" directory. # For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory from subprocess import check_output print(check_output(["ls", "../input"]).decode("utf8")) import glob from multiprocessing import Process import os import re import numpy as np import PIL from PIL import Image import os from keras.preprocessing.image import ImageDataGenerator from keras.models import Sequential from keras.layers import Convolution2D, MaxPooling2D from keras.layers import Activation, Dropout, Flatten, Dense import numpy as np from PIL import Image from keras import backend as K def natural_key(string_): """ Define sort key that is integer-aware """ return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] def norm_image(img): """ Normalize PIL image Normalizes luminance to (mean,std)=(0,1), and applies a [1%, 99%] contrast stretch """ img_y, img_b, img_r = img.convert('YCbCr').split() img_y_np = np.asarray(img_y).astype(float) img_y_np /= 255 img_y_np -= img_y_np.mean() img_y_np /= img_y_np.std() scale = np.max([np.abs(np.percentile(img_y_np, 1.0)), np.abs(np.percentile(img_y_np, 99.0))]) img_y_np = img_y_np / scale img_y_np = np.clip(img_y_np, -1.0, 1.0) img_y_np = (img_y_np + 1.0) / 2.0 img_y_np = (img_y_np * 255 + 0.5).astype(np.uint8) img_y = Image.fromarray(img_y_np) img_ybr = Image.merge('YCbCr', (img_y, img_b, img_r)) img_nrm = img_ybr.convert('RGB') return img_nrm def resize_image(img, size): """ Resize PIL image Resizes image to be square with sidelength size. Pads with black if needed. """ # Resize n_x, n_y = img.size if n_y > n_x: n_y_new = size n_x_new = int(size * n_x / n_y + 0.5) else: n_x_new = size n_y_new = int(size * n_y / n_x + 0.5) img_res = img.resize((n_x_new, n_y_new), resample=PIL.Image.BICUBIC) # Pad the borders to create a square image img_pad = Image.new('RGB', (size, size), (128, 128, 128)) ulc = ((size - n_x_new) // 2, (size - n_y_new) // 2) img_pad.paste(img_res, ulc) return img_pad def prep_images(paths, out_dir): """ Preprocess images Reads images in paths, and writes to out_dir """ for count, path in enumerate(paths): if count % 100 == 0: print(path) img = Image.open(path) img_nrm = norm_image(img) img_res = resize_image(img_nrm, SIZE) basename = os.path.basename(path) path_out = os.path.join(out_dir, basename) img_res.save(path_out) def process(): """Main program for running from command line""" # Get the paths to all the image files train_cats = sorted(glob.glob(os.path.join(TRAIN_DIR, 'cat*.jpg')), key=natural_key) train_dogs = sorted(glob.glob(os.path.join(TRAIN_DIR, 'dog*.jpg')), key=natural_key) test_all = sorted(glob.glob(os.path.join(TEST_DIR, '*.jpg')), key=natural_key) # Make the output directories base_out = os.path.join(BASE_DIR, 'input{}'.format(SIZE)) train_dir_out = os.path.join(base_out, 'train') test_dir_out = os.path.join(base_out, 'test') os.makedirs(train_dir_out, ) os.makedirs(test_dir_out) # Preprocess the training files procs = dict() procs[1] = Process(target=prep_images, args=(train_cats, train_dir_out, )) procs[1].start() procs[2] = Process(target=prep_images, args=(train_dogs, train_dir_out, )) procs[2].start() procs[3] = Process(target=prep_images, args=(test_all, test_dir_out, )) procs[3].start() procs[1].join() procs[2].join() procs[3].join() # Any results you write to the current directory are saved as output. '''This script goes along the blog post "Building powerful image classification models using very little data" from blog.keras.io. It uses data that can be downloaded at: https://www.kaggle.com/c/dogs-vs-cats/data In our setup, we: - created a data/ folder - created train/ and validation/ subfolders inside data/ - created cats/ and dogs/ subfolders inside train/ and validation/ - put the cat pictures index 0-999 in data/train/cats - put the cat pictures index 1000-1400 in data/validation/cats - put the dogs pictures index 12500-13499 in data/train/dogs - put the dog pictures index 13500-13900 in data/validation/dogs So that we have 1000 training examples for each class, and 400 validation examples for each class. In summary, this is our directory structure: ``` data/ train/ dogs/ dog001.jpg dog002.jpg ... cats/ cat001.jpg cat002.jpg ... validation/ dogs/ dog001.jpg dog002.jpg ... cats/ cat001.jpg cat002.jpg ... ``` ''' # http://stackoverflow.com/questions/38155836/data-augmentation-image-data-generator-keras-semantic-segmentation def img_to_array(infilename, dim_ordering='default'): """ Create array from image. """ img = Image.open( os.path.join(os.path.dirname(__file__),infilename) ).convert('L') img.load() if dim_ordering == 'default': dim_ordering = K.image_dim_ordering() if dim_ordering not in ['th', 'tf']: raise Exception('Unknown dim_ordering: ', dim_ordering) # image has dim_ordering (height, width, channel) x = np.asarray(img, dtype='float32')/255.0 if len(x.shape) == 3: if dim_ordering == 'th': x = x.transpose(2, 0, 1) elif len(x.shape) == 2: if dim_ordering == 'th': x = x.reshape((1, x.shape[0], x.shape[1])) else: x = x.reshape((x.shape[0], x.shape[1], 1)) else: raise Exception('Unsupported image shape: ', x.shape) return x def image_generator(list_of_files): """ Create array from image. """ while True: filename = np.random.choice(list_of_files) try: img = img_to_array(filename) except: return # 0 DOG # 1 CAT yield (img, [1 if 'dog' in filename else 0]) # yield (img, {'output': 0 if 'dog' in filename else 1}) def group_by_batch(dataset, batch_size): """ Create array from image. """ while True: try: sources, targets = zip(*[next(dataset) for i in xrange(batch_size)]) batch = (np.stack(sources), np.stack(targets)) yield batch except: return def load_dataset(files, batch_size): """ Create array from image. """ generator = image_generator(files) generator = group_by_batch(generator, batch_size) return generator # Processing parameters SIZE = 224 # for ImageNet models compatibility TEST_DIR = '../input/test/' TRAIN_DIR = '../input/train/' BASE_DIR = '.' process() TRAIN_DIR = 'data224/train/' TRAIN_DOG_DIR = 'data224/train/dog' TRAIN_CAT_DIR = 'data224/train/cat' VALIDATION_DIR = 'data224/validation/' VALIDATION_DOG_DIR = 'data224/validation/dog' VALIDATION_CAT_DIR = 'data224/validation/cat' train_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)] # use this for full dataset train_dogs = [TRAIN_DOG_DIR+'/'+i for i in os.listdir(TRAIN_DOG_DIR) ] train_cats = [TRAIN_CAT_DIR+'/'+i for i in os.listdir(TRAIN_CAT_DIR) ] test_images = [VALIDATION_DIR+i for i in os.listdir(VALIDATION_DIR)] # use this for full dataset test_dogs = [VALIDATION_DOG_DIR+'/'+i for i in os.listdir(VALIDATION_DOG_DIR) ] test_cats = [VALIDATION_CAT_DIR+'/'+i for i in os.listdir(VALIDATION_CAT_DIR) ] # dimensions of our images. img_width, img_height = 224, 224 model = Sequential() model.add(Convolution2D(32, 3, 3, input_shape=(img_width, img_height, 1))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(32, 3, 3)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Convolution2D(64, 3, 3)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(64)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) list_train = train_dogs + train_cats model.fit_generator(load_dataset(list_train, 30), samples_per_epoch=30, nb_epoch=5000) list_test = test_dogs + test_cats scores = model.evaluate_generator(load_dataset(list_test, 5), 300, max_q_size=10, nb_worker=1, pickle_safe=True) print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100)) model_json = model.to_json() with open("model.json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights("model.h5") print("Saved model to disk")