{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.applications import ResNet50\nfrom keras.applications import imagenet_utils\nfrom keras.preprocessing.image import img_to_array\nfrom keras.preprocessing.image import load_img\nfrom sklearn.preprocessing import LabelEncoder\nimport progressbar\nimport random\nimport cv2\n#import matplotlib.pyplot as plt\n#import seaborn as sns\nimport h5py\nimport gc\n%matplotlib inline\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\n\nimage_types = (\".jpg\", \".jpeg\", \".png\", \".bmp\", \".tif\", \".tiff\")\n\n\ndef list_images(basePath, contains=None):\n    # return the set of files that are valid\n    return list_files(basePath, validExts=image_types, contains=contains)\n\n\ndef list_files(basePath, validExts=None, contains=None):\n    # loop over the directory structure\n    for (rootDir, dirNames, filenames) in os.walk(basePath):\n        # loop over the filenames in the current directory\n        for filename in filenames:\n            # if the contains string is not none and the filename does not contain\n            # the supplied string, then ignore the file\n            if contains is not None and filename.find(contains) == -1:\n                continue\n\n            # determine the file extension of the current file\n            ext = filename[filename.rfind(\".\"):].lower()\n\n            # check to see if the file is an image and should be processed\n            if validExts is None or ext.endswith(validExts):\n                # construct the path to the image and yield it\n                imagePath = os.path.join(rootDir, filename)\n                yield imagePath\n\ndef resize(image, width=None, height=None, inter=cv2.INTER_AREA):\n    # initialize the dimensions of the image to be resized and\n    # grab the image size\n    dim = None\n    (h, w) = image.shape[:2]\n\n    # if both the width and height are None, then return the\n    # original image\n    if width is None and height is None:\n        return image\n\n    # check to see if the width is None\n    if width is None:\n        # calculate the ratio of the height and construct the\n        # dimensions\n        r = height / float(h)\n        dim = (int(w * r), height)\n\n    # otherwise, the height is None\n    else:\n        # calculate the ratio of the width and construct the\n        # dimensions\n        r = width / float(w)\n        dim = (width, int(h * r))\n\n    # resize the image\n    resized = cv2.resize(image, dim, interpolation=inter)\n\n    # return the resized image\n    return resized","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class HDF5DatasetWriter:\n    def __init__(self, dims, outputPath, dataKey = 'images', bufSize=1000):\n        if os.path.exists(outputPath):\n            raise ValueError('The supplied \"outputPath\" already exists. Manually delete the file before continuing.',outputPath)\n            \n        self.db = h5py.File(outputPath, mode='w')\n        self.data = self.db.create_dataset(dataKey, dims, dtype='float')\n        self.labels = self.db.create_dataset('labels', (dims[0],), dtype='int')\n        \n        self.bufSize = bufSize\n        self.buffer = {'data':[], 'labels':[]}\n        self.idx = 0\n        \n    def add(self, rows, labels):\n        self.buffer['data'].extend(rows)\n        self.buffer['labels'].extend(labels)\n        \n        if len(self.buffer['data']) >= self.bufSize:\n            self.flush()\n    \n    def flush(self):\n        i = self.idx + len(self.buffer['data'])\n        self.data[self.idx:i] = self.buffer['data']\n        self.labels[self.idx:i] = self.buffer['labels']\n        self.idx = i\n        \n        self.buffer = {'data':[], 'labels':[]}\n        \n    def storeClassLabels(self, classLabels):\n        \n        dt = h5py.special_dtype(vlen=str)\n        labelSet = self.db.create_dataset('label_name', (len(classLabels),), dtype = dt)\n        labelSet[:] = classLabels\n        \n    def close(self):\n        if len(self.buffer['data']) > 0:\n            self.flush()\n        \n        self.db.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = '../input/dogs-vs-cats-redux-kernels-edition/train/train/'\nfinal_test_path = '../input/dogs-vs-cats-redux-kernels-edition/test/test/'\n#train_img=[train_path+'/'+i for i in os.listdir(train_path)]\ntrain_img_paths = list(list_images(train_path))\nfinal_test_img_paths = list(list_images(final_test_path))\nlen(train_img_paths), len(final_test_img_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainLabels = []\ncount_rej = 0\nfor p in train_img_paths:\n    if p.split(os.path.sep)[-1].split('.')[-1] == 'jpg':\n        trainLabels.append(p.split(os.path.sep)[-1].split('.')[0])\n    else:\n        count_rej +=1\nprint(len(trainLabels),'\\n', np.unique(trainLabels),'\\n',count_rej)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\ntrainLabels = le.fit_transform(trainLabels)\nprint(len(trainLabels),'\\n', np.unique(trainLabels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = ResNet50(include_top=False, weights='imagenet')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset = HDF5DatasetWriter((len(train_img_paths), 2048), 'resnet_dogs_vs_cats_features.hdf5', dataKey='features', bufSize = 1000)\ndataset.storeClassLabels(le.classes_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('/kaggle/working/')\nbs = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import MaxPooling2D, Flatten\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"widgets=['Extracting Features:', progressbar.Percentage(), ' ', progressbar.Bar(), ' ', progressbar.ETA()]\npbar = progressbar.ProgressBar(max_value=len(train_img_paths), widgets=widgets).start()\nfor i in np.arange(0,len(train_img_paths), bs):\n    batchPaths = train_img_paths[i:i+bs]\n    batchLabels = trainLabels[i:i+bs]\n    batchImages=[]\n    for j,imagePath in enumerate(batchPaths):\n        image = load_img(imagePath, target_size=(224,224))\n        image = img_to_array(image)\n        image = np.expand_dims(image, axis=0)\n        image = imagenet_utils.preprocess_input(image)\n        batchImages.append(image)\n    batchImages = np.vstack(batchImages)\n    features = model.predict(batchImages, batch_size=bs)\n    features = K.constant(features)\n    features = MaxPooling2D(pool_size=(7,7))(features)\n    features = Flatten()(features)\n    features = K.eval(features)\n#     print(features.shape, type(features))\n    dataset.add(features, batchLabels)\n    pbar.update(i)\n\ndatset.close()\npbar.finish","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from IPython.display import FileLink\n# FileLink('/kaggle/working/resnet_dogs_vs_cats_features.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# a= [np.arange(224*224*3).reshape(1,224,224,3)]\n# a.append(np.arange(224*224*3).reshape(1,224,224,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.vstack(a)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# os.remove('/kaggle/working/resnet_dogs_vs_cats_features.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}