{"cells":[{"metadata":{"_uuid":"d8d70e3e29a41aa5d537efdb0c11dfd7cc8c2f52"},"cell_type":"markdown","source":"https://www.kaggle.com/c/dogs-vs-cats-redux-kernels-edition\n\nDogs vs. Cats classification problem"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import os, cv2, random\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nfrom matplotlib import ticker\nimport seaborn as sns\n%matplotlib inline \n\nimport warnings\nwarnings.filterwarnings(\"ignore\",category=DeprecationWarning)\nwarnings.filterwarnings(\"ignore\",category=FutureWarning)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3689c5f6bf3a9339549b235430d707eaa1077a4"},"cell_type":"code","source":"import keras\n\nprint(\"Keras -V: {}\".format(keras.__version__))\nprint(\"OpenCV -V: {}\".format(cv2.__version__))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ee91372b198dd0487f2b45b1d026658969331336"},"cell_type":"code","source":"from keras.optimizers import SGD, RMSprop, Adam\n\n# config\nFORCE_CACHE = False #True\n\nTRAIN_DIR = '../input/train/'\nTEST_DIR = '../input/test/'\nCACHE_DIR = '../working/'\nCACHE_FILE = 'cache.hdf5'\n\nRANDOM_SEED = 1980\nCHUNK_SIZE = 2000\nIMG_ROWS = 128\nIMG_COLS = 128\nIMG_CHANNELS = 3\nPIXEL_DEPTH = 255\n\nNB_EPOCH = 50\nBATCH_SIZE = 128\nVERBOSE = 1\nOPTIMIZER = RMSprop() \nN_HIDDEN = 128\nNB_CLASSES = 2\nVALIDATION_SPLIT=0.2\nDROPOUT = 0.3\n\nINPUT_SHAPE = (IMG_CHANNELS, IMG_ROWS, IMG_COLS)\nCACHE_FILE = CACHE_FILE[:-5] + str(IMG_ROWS) + '.hdf5'\nCACHE = os.path.join(CACHE_DIR, CACHE_FILE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a03dfc61677a59e43a98bf31b334d32b7d40fa8a","collapsed":true},"cell_type":"code","source":"from keras.utils import np_utils\nfrom itertools import repeat\nfrom sklearn.model_selection import train_test_split\n\nnp.random.seed(RANDOM_SEED)  # for reproducibility\n\n# get the data\ntrain_images = [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR)]\ntrain_dogs =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'dog' in i]\ntrain_cats =   [TRAIN_DIR+i for i in os.listdir(TRAIN_DIR) if 'cat' in i]\ntest_images =  [TEST_DIR+i for i in os.listdir(TEST_DIR)]\n\ntrain_images = train_dogs + train_cats\n\n# prepare target y\nlabels = list(repeat(1, len(train_dogs))) + list(repeat(0, len(train_cats)))\nlabels = np_utils.to_categorical(labels, NB_CLASSES)\n\n# training/test Split\nX_train, X_test, Y_train, Y_test = train_test_split(train_images, labels,\n                                                    test_size=VALIDATION_SPLIT,\n                                                    random_state=RANDOM_SEED)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f5fac60546b0843f6ce0122a75f919c29138fd6f"},"cell_type":"markdown","source":"Define generator due to the large size."},{"metadata":{"trusted":true,"_uuid":"77f10730c8b22b06049ab66d540890a7791e95b9"},"cell_type":"code","source":"import h5py\n\ndef read_images(image_files):\n    #cv2.imread return BGR image\n    images = [cv2.imread(file, cv2.IMREAD_COLOR) for file in image_files]\n    #blobFromImages(swapRB=True) return RGB image\n    data = cv2.dnn.blobFromImages(images, size=(IMG_ROWS,IMG_COLS), \n                                  scalefactor=1./PIXEL_DEPTH,\n                                  swapRB=True, crop=False)\n    \n    #assert data is not None\n    return data #RGB images\n\ndef create_hdf5_cache_file(chunksize, folder = './', file_path = 'data.hdf5'):\n    if not os.path.isdir(folder):\n        os.makedirs(folder)\n\n    with h5py.File(os.path.join(folder, file_path), \"w\") as f:\n        x_list = [X_train, X_test]\n        y_list = [Y_train, Y_test]\n        group_list = ['train', 'test']\n\n        for x, y, group in zip(x_list, y_list, group_list):\n            print('Start processing subgroup: {}'.format(group))\n            grp = f.create_group(group)\n            count = len(x)\n            dx = grp.create_dataset(\"dx\",(count,)+INPUT_SHAPE,'float32')\n            grp[\"dy\"] = y\n            for i in range(0, count, chunksize):\n                print('Processing x: {} of {}'.format(i, count))\n                dx[i:i+chunksize,:,:,:] = read_images(x[i:i+chunksize])\n            print('Processed subgroup: {}'.format(group))\n\n    print(\"Completed writing hdf5 file: {}{}\".format(folder,file_path))\n\nif FORCE_CACHE or not os.path.isfile(CACHE):\n    create_hdf5_cache_file(CHUNK_SIZE, CACHE_DIR, CACHE_FILE)\nelse:\n    print(\"Make use of original hdf5 file: {}{}\".format(CACHE_DIR, CACHE_FILE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d2d9977a0a1fe3cc9e1d867511507c7fc0f9400f"},"cell_type":"code","source":"from keras.utils import HDF5Matrix\n\nclass BatchDataGenerator(keras.utils.Sequence):\n    def __init__(self, hdf5_file, group, batch_size, start=0, end=None):\n        self.file = hdf5_file\n        self.group = group\n        self.batch_size = batch_size\n        self.start = start\n        if end is None:\n            with h5py.File(hdf5_file, \"r\") as f:\n                self.end = f[group][\"dy\"].len()\n        else:\n            self.end = end\n            \n        self.length = self.end - self.start\n        self.start_list = list(range(self.start, self.end, self.batch_size))\n        self.end_list = [j+self.batch_size-1 for j in self.start_list]\n        self.end_list[-1] = min(self.end_list[-1], self.end)       \n    \n    def __len__(self):\n        return int(np.ceil(self.length / float(self.batch_size)))\n    \n    def __getitem__(self, idx):\n        params = {'start': self.start_list[idx],\n                  'end': self.end_list[idx]}\n        batch_x = HDF5Matrix(self.file, self.group+'/dx', **params)\n        batch_y = HDF5Matrix(self.file, self.group+'/dy', **params)\n        return batch_x, batch_y\n\ndef generate_full_data(hdf5_file, group, start=0, end=None):\n    with h5py.File(hdf5_file, \"r\") as f:\n        if end is None:\n            end = f[group][\"dy\"].len()\n        x = f[group][\"dx\"][start:end]\n        y = f[group][\"dy\"][start:end]\n    \n    return x, y\n\ndef get_train_split_indices(hdf5_file, group, split_ratio):\n    with h5py.File(hdf5_file, \"r\") as f:\n        data_size = f[group][\"dy\"].len()\n    \n    return 0, int(data_size*(1-split_ratio)), int(data_size*(1-split_ratio))+1, data_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2d13494289884d597d03f56cb0b227bb2ac8ad6","collapsed":true},"cell_type":"code","source":"train_st,train_ed,valid_st,valid_ed = get_train_split_indices(CACHE,'train',VALIDATION_SPLIT)\n\ntraining_generator = BatchDataGenerator(CACHE,'train',BATCH_SIZE,train_st,train_ed)\nvalidation_generator = BatchDataGenerator(CACHE,'train',BATCH_SIZE,valid_st,valid_ed)\n\nvalidation_data = generate_full_data(CACHE,'train',valid_st,valid_ed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d25d34065735aafccbd493efdf9eb8d5b5c53bab"},"cell_type":"code","source":"from keras import backend as K\nfrom keras.models import Sequential\nfrom keras.layers.convolutional import Conv2D\nfrom keras.layers.convolutional import MaxPooling2D\nfrom keras.layers.core import Dense, Dropout, Activation, Reshape\nfrom keras.layers.core import Flatten\n\n# model\nK.set_image_dim_ordering(\"th\")\n#define the convnet \nclass myNet:\n    @staticmethod\n    def build(input_shape, classes):\n        model = Sequential()\n        model.add(Conv2D(20, kernel_size=5, padding=\"same\", \n                         input_shape=input_shape, activation='relu'))\n        model.add(MaxPooling2D(pool_size=(2,2)))\n        model.add(Conv2D(50, kernel_size=5, padding=\"same\", activation='relu'))\n        model.add(MaxPooling2D(pool_size=(2,2)))\n        model.add(Dropout(DROPOUT))\n        model.add(Flatten())\n        model.add(Dense(500, activation='relu'))\n        model.add(Dropout(DROPOUT))\n        model.add(Dense(classes, activation='sigmoid'))\n        return model\n\n# initialize the optimizer and model\nmodel = myNet.build(input_shape=INPUT_SHAPE, classes=NB_CLASSES)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef83efa0dce517a447ad23ff01b825c64b579ec2"},"cell_type":"code","source":"from keras.callbacks import TensorBoard, EarlyStopping\n\nmodel.compile(loss='binary_crossentropy',\n              optimizer=OPTIMIZER,\n              metrics=['accuracy'])\n\nesCallBack = EarlyStopping(monitor='val_loss', patience=20,  \n                           min_delta = 0, verbose = 0, mode = 'min')\nhistory = model.fit_generator(training_generator,\n                              epochs=NB_EPOCH,\n                              validation_data=validation_data,\n                              verbose=VERBOSE,\n                              workers=4,\n                              callbacks=[esCallBack])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80bf59e9185ff09d20a4a0e2fb60e970495e0e60","scrolled":true},"cell_type":"code","source":"X_test_data, Y_test_data = generate_full_data(CACHE,'test')\n\nscore = model.evaluate(X_test_data, Y_test_data, verbose=VERBOSE)\nprint(\"\\nTest score:\", score[0])\nprint('Test accuracy:', score[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"6be68d7c13a999e0401874fdce00939650dbc1c2"},"cell_type":"code","source":"# list all data in history\nprint(history.history.keys())\n# summarize history for accuracy\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n# summarize history for loss\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0229ed4b7d3a2e88ab9bf7d626f02c92164c9b10"},"cell_type":"markdown","source":"X_submission = read_images(test_images)\n\nsubmission = model.predict(X_submission,verbose=VERBOSE)"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"91012cd0cc526aa7f3ab8c8b4046ed98cda86e5c"},"cell_type":"markdown","source":"df_sub = pd.DataFrame({'id':test_images})\ndf_sub['id'] = df_sub['id'].str.extract(r'(\\d+)').apply(pd.to_numeric)\ndf_sub['label'] = pd.DataFrame(submission[:,1])"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"2bd74c36bbdb5b4e1bdcb695c70f613e22ab6b53"},"cell_type":"markdown","source":"df_sub.to_csv('../working/output.csv', index=False)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}