{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport bcolz\nimport random\nfrom tqdm import tqdm\nfrom glob import glob\nfrom keras.preprocessing.image import load_img\nfrom keras.preprocessing.image import img_to_array\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/dog-breed-identification/labels.csv')\ndf_test = pd.read_csv('../input/dog-breed-identification/sample_submission.csv')\ntrain_files = glob('../input/dog-breed-identification/train/*.jpg')\ntest_files = glob('../input/dog-breed-identification/test/*.jpg')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"targets_series = pd.Series(df_train['breed'])\ntargets_names = df_train['breed'].tolist()\none_hot = pd.get_dummies(targets_series, sparse = True)\none_hot_labels = np.asarray(one_hot)\nprint(targets_names[0:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"im_size = 300\nx_train_raw = bcolz.zeros((0,im_size,im_size,3),np.float32)\nx_val_raw = bcolz.zeros((0,im_size,im_size,3),np.float32)\ny_train = []\ny_val = []\ny_train_name = []\ny_val_name = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_raw1 = bcolz.zeros((1,4,2,3),np.float32)\nprint(x_train_raw1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i = 0 \nfor f in tqdm(train_files):\n    image = load_img(train_files[i].format(f), target_size=(im_size, im_size))\n    image = img_to_array(image)\n    label = one_hot_labels[i]\n    names = targets_names[i]\n    if random.randint(1,101) < 80: \n        x_train_raw.append(image)\n        y_train.append(label)\n        y_train_name.append(names)\n    else:\n        x_val_raw.append(image)\n        y_val.append(label)\n        y_val_name.append(names)\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_train_raw.shape, x_val_raw.shape)\nprint(len(y_val),len(y_val), len(y_val_name))\nplt.imshow(x_train_raw[2]/255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dummy data for testing\ndummyX = x_train_raw[0:16]\ndummyY = y_train[0:16]\ndummyN = y_train_name[0:16]\nprint(dummyN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### defining fucntion img_trans"},{"metadata":{"trusted":true},"cell_type":"code","source":"def img_trans(img_data):\n    im_size = img_data.shape\n    nd1 = int(np.floor(im_size[0]*0.8))\n    nd2 = int(np.floor(im_size[0]*0.8))\n    cnd1 = int(np.floor((im_size[0]-nd1)*0.5))\n    cnd2 = int(np.floor((im_size[1]-nd2)*0.5))\n    k = random.randint(0,4)\n    if k == 0:\n        new_image = img_data\n        new_image[0:cnd1*2] = [0,0,0]\n        new_image[:,0:cnd2*2] = [0,0,0]\n    if k == 1:\n        new_image = img_data[im_size[0]-nd1:im_size[0],0:nd2]\n    if k == 2:\n        new_image = img_data[0:nd1,im_size[1]-nd2:im_size[1]]\n    if k == 3:\n        new_image = img_data[im_size[0]-nd1:im_size[0],im_size[1]-nd2:im_size[1]]\n    if k == 4:\n        new_image = img_data[cnd1:im_size[0]-cnd1,cnd2:im_size[1]-cnd2]\n    hflip = random.randint(1,101)\n    if hflip > 50:\n        new_image = np.flip(new_image,1)\n    vflip = random.randint(1,101)\n    if vflip > 50:\n        new_image = np.flip(new_image,0)\n    \n    return new_image","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### function for New data generation and augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augmentation(image_data, image_labels, image_names):\n    x_aug = bcolz.zeros((0,im_size,im_size,3),np.float32)\n    y_aug = []\n    yN_aug = []\n    count = len(image_labels)\n    i=0\n    for i in range(count):\n        x_aug.append(image_data[i]/255.)\n        y_aug.append(image_labels[i])\n        yN_aug.append(image_names[i])\n        x_aug.append(img_trans(image_data[i]/255.))\n        y_aug.append(image_labels[i])\n        yN_aug.append(image_names[i])\n    return x_aug, y_aug, yN_aug","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dumX_aug, dumY_aug, dumN_aug = data_augmentation(dummyX, dummyY, dummyN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### display augmented data - only for first 2 pairs of images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_augdata(image_data, image_names, batch_size):\n    image_data1 = image_data[0:batch_size*2]\n    image_names1 = image_names[0:batch_size*2]\n    fig, axes = plt.subplots(batch_size, batch_size, figsize=(20,20))\n    axes = axes.flatten()\n    i=0\n    for img, ax in zip(image_data1, axes):\n        ax.set_title(image_names1[i], fontsize=36)\n        ax.imshow(img)\n        ax.set_xticks(())\n        ax.set_yticks(())\n        i=i+1\n    plt.tight_layout()\ndisplay_augdata(dumX_aug, dumN_aug, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def DataGen(train_data_X, train_data_Y, train_data_N, im_size, batch_size, shuffle=True, data_aug = True, test = False):\n    nobs = len(train_data_N)\n    iter_times = nobs/batch_size\n    train_data_X.list_IDs = np.arange(0,data.shape[0])\n    if shuffle=True:\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameters\nparams_trn = {\n          'im_size': im_size,\n          'batch_size': batch_size,\n          'shuffle': True,\n          'data_augment' : True,\n          'test' : False\n         }\nparams_val = {\n          'im_size': im_size,\n          'batch_size': batch_size,\n          'shuffle': True,\n          'data_augment' : False,\n          'test' : False\n         }\n\n# Generators\ntraining_generator = DataGenerator(x_train_raw, y_train_raw, **params_trn)\nvalidation_generator = DataGenerator(x_val_raw, y_val_raw, **params_val)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}