{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport feather\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas()\n# from tqdm import tqdm\n\n# pd.options.display.max_rows = 999\n# pd.options.display.max_columns = 999\nimport glob\ndef get_path(str, first=True, parent_dir='../input/**/'):\n    res_li = glob.glob(parent_dir+str)\n    return res_li[0] if first else res_li","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/dogs-vs-cats-redux-kernels-edition/'\nevals = pd.read_csv('../input/dvc-prepare-evalset/evals.csv')\nevals.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2f2bd0a567f12e8778a357e29c8280fa5d2c7ad"},"cell_type":"markdown","source":"## Load & Resize Image\n- cv2.imread with wrong path returns None other than raises Error\n- cv2.imread reads images in BGR channels\n- cv2.resize can set different interpolations"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"052c6b0c7448a11076309913be8f2116d9e3ac3c"},"cell_type":"code","source":"H, W, C = 128, 128, 3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82c588bf835022758a25592ea01aec403fc1ff28"},"cell_type":"code","source":"import cv2\ntmp = cv2.cvtColor(\n    cv2.imread(DATA_DIR+'train/cat.0.jpg', cv2.IMREAD_COLOR), \n    cv2.COLOR_BGR2RGB\n).astype('float')/255.\nprint('shape', tmp.shape)\ntmp_rsz = cv2.resize(tmp, (H, W), interpolation=cv2.INTER_NEAREST)\nprint('resized', tmp_rsz.shape)\nplt.figure(figsize=[12, 8])\nplt.subplot(1, 2, 1)\nplt.imshow(tmp); plt.title('Original'); \nplt.subplot(1, 2, 2)\nplt.imshow(tmp_rsz); plt.title('Resized'); ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3015e1f3ca740a1de38dff4bf4ef3761f5da3ee9"},"cell_type":"markdown","source":"## Image Normalization"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d0d760bc32756ca37e4dfd19438cbafa7e470640"},"cell_type":"code","source":"from skimage import exposure","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb8016911c610a2e03eaacc09330782796711115"},"cell_type":"code","source":"tmp = tmp_rsz.copy()\ntmp_bri_norm = (tmp - np.mean(tmp))/np.std(tmp)\ntmp_bri_norm -= tmp_bri_norm.min()\ntmp_bri_norm /= tmp_bri_norm.max()\ntmp_hist_eq = exposure.equalize_adapthist(tmp.copy(), clip_limit=0.01)\n\ndef plot_hist(img):\n    for i in range(3):\n        sns.distplot(img[:, :, i].ravel());\n        plt.legend(['R', 'G', 'B']);\n\nplt.figure(figsize=[22, 18])\nplt.subplot(1, 4, 1)\nplt.imshow(tmp); plt.title('Resized'); \nplt.subplot(1, 4, 2)\nplt.imshow(tmp_bri_norm); plt.title('Bright_normalized'); \nplt.subplot(1, 4, 3)\nplt.imshow(tmp_hist_eq); plt.title('Histogram_equalized limit 0.01'); \nplt.subplot(1, 4, 4)\nclip_limit = 0.1\nplt.imshow(exposure.equalize_adapthist(tmp, clip_limit=clip_limit)); \nplt.title(f'Histogram_equalized limit {clip_limit}'); \n\nplt.figure(figsize=[22, 4])\nplt.subplot(1, 4, 1)\nplot_hist(tmp); plt.title('Resized'+' Hist'); \nplt.subplot(1, 4, 2)\nplot_hist(tmp_bri_norm); plt.title('Bright_normalized'+' Hist'); \nplt.subplot(1, 4, 3)\nplot_hist(tmp_hist_eq); plt.title(f'Histogram_equalized limit 0.01'+' Hist'); \nplt.subplot(1, 4, 4)\nplot_hist(exposure.equalize_adapthist(tmp, clip_limit=0.03)); \nplt.title(f'Histogram_equalized limit {clip_limit}'+' Hist'); ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80c0da215c255c6a40d8c9d366ad9907da7cdc61"},"cell_type":"markdown","source":"## Data Generation&Augmentation\n- https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html\n- imagedatagenerator.flow_from_dir can be re-set after initialization"},{"metadata":{"trusted":true,"_uuid":"d145f6546d894f30c0201f4ec4551fffbff37054"},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras.preprocessing import image","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f7a270b0fa60eb8023c8da0670456fcdbd3783c3"},"cell_type":"code","source":"imgGen = image.ImageDataGenerator(\n    #featurewise_center=True,\n    #featurewise_std_normalization=True,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    channel_shift_range=0.2,\n    horizontal_flip=True,\n    vertical_flip=True,\n    rescale=1./255,\n)\nimgGen_test = image.ImageDataGenerator(rescale=1./255,)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7d635e99834076bc96b4371a75967bc4e1d830f"},"cell_type":"code","source":"train_gen = imgGen.flow_from_directory(\n    DATA_DIR+'train/',\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=32,\n)\n\nvalid_fold = 0\ntrain_gen.class_indices = {'dog': 0, 'cat': 1}\nmask = (evals['is_test']==0) & (evals['eval_set']!=valid_fold)\ntrain_gen.filenames = evals.loc[mask, 'img_id'].apply(lambda x: x+'.jpg').values.tolist()\ntrain_gen.classes = evals.loc[mask, 'target'].values\ntrain_gen.class_mode = 'binary'\ntrain_gen.samples = len(evals.loc[mask, 'target'].values)\ntrain_gen.n = len(evals.loc[mask, 'target'].values)\ntrain_gen.num_classes = 2\n\nfor bx, by in train_gen:\n    break\nprint('targets(is_cat?)', [by[i] for i in range(12)])\nplt.figure(figsize=[12, 8])\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    plt.imshow(bx[i]);\n    plt.axis('off');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7bd2ea70d5691aaad2d10303e82bd522134a3cd"},"cell_type":"code","source":"test_gen = imgGen_test.flow_from_directory(\n    DATA_DIR+'test/',\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=32,\n)\n\nvalid_fold = 0\n# test_gen.class_indices = {'dog': 0, 'cat': 1}\nmask = (evals['is_test']==1)\ntest_gen.filenames = evals.loc[mask, 'img_id'].apply(lambda x: x+'.jpg').values.tolist()\ntest_gen.classes = evals.loc[mask, 'target'].values\ntest_gen.class_mode = 'binary'\ntest_gen.samples = len(evals.loc[mask, 'target'].values)\ntest_gen.n = len(evals.loc[mask, 'target'].values)\ntest_gen.num_classes = 2\n\nfor bx, by in test_gen:\n    break\nprint('targets(is_cat?)', [by[i] for i in range(12)])\nplt.figure(figsize=[12, 8])\nfor i in range(12):\n    plt.subplot(3, 4, i+1)\n    plt.imshow(bx[i]);\n    plt.axis('off');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"55c03ea3dc7e36c9603f0e99ead94b2c4cf2163c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0adaa688202c9440d19570ea3053a87ab02d2256"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"e84f4539ece2dd3ee3782e8cf45bfed4ee80b054"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ee029da422ef8454d9d60cc585e373a98e1ef1a3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}