{"cells":[{"metadata":{},"cell_type":"markdown","source":"Image augmentaion library인 imgaug를 사용해보는 예제입니다.\n\n**Reference **  \nhttps://www.kaggle.com/fulrose/3rd-ml-month-car-model-classification-baseline  \nhttps://www.kaggle.com/tmheo74/3rd-ml-month-car-image-cropping-updated-7-10  \nhttps://www.kaggle.com/easter3163/3rd-ml-month-keras-efficientnet  \nhttps://www.kaggle.com/seriousran/what-0-95121-model-failed  \nhttps://www.kaggle.com/janged/3rd-ml-month-xception-stratifiedkfold-ensemble  \nhttps://www.kaggle.com/ratan123/aptos-2019-keras-baseline  \nhttps://github.com/aleju/imgaug"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install git+https://github.com/qubvel/efficientnet","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport warnings\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\n\nimport cv2\nimport PIL\nfrom PIL import ImageOps, ImageFilter, ImageDraw\n\nfrom keras import backend as K\nfrom keras.models import Sequential, Model\nfrom keras.layers import Dense, Dropout, Flatten, Activation, Conv2D, GlobalAveragePooling2D\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, EarlyStopping, CSVLogger\n\nfrom keras.preprocessing.image import ImageDataGenerator\n\nfrom efficientnet.keras import EfficientNetB3\n\nfrom sklearn.model_selection import train_test_split\n\nfrom imgaug import augmenters as iaa\nimport imgaug as ia\n\nfrom keras.utils import Sequence, to_categorical\n\nfrom sklearn.utils import shuffle\n\n\nwarnings.filterwarnings(action='ignore')\n\nK.image_data_format()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameter\napplication = EfficientNetB3\nimg_size=300\nnet_name='efficientnetb3'\nlearning_rate=0.001\nmin_learning_rate=0.00001\npatience=4\n\nepochs = 100\nbatch_size = 32","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def recall_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\ndef precision_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_boxing_img(img_name, margin=16) :\n    if img_name.split('_')[0] == \"train\" :\n        PATH = TRAIN_IMG_PATH\n        data = df_train\n    elif img_name.split('_')[0] == \"test\" :\n        PATH = TEST_IMG_PATH\n        data = df_test\n        \n    img = PIL.Image.open(os.path.join(PATH, img_name))\n    pos = data.loc[data[\"img_file\"] == img_name, \\\n                   ['bbox_x1','bbox_y1', 'bbox_x2', 'bbox_y2']].values.reshape(-1)\n\n    width, height = img.size\n    x1 = max(0, pos[0] - margin)\n    y1 = max(0, pos[1] - margin)\n    x2 = min(pos[2] + margin, width)\n    y2 = min(pos[3] + margin, height)\n    \n    if abs(pos[2] - pos[0]) > width or abs(pos[3] - pos[1]) > height:\n        print(f'{img_name} is wrong bounding box, img size: {img.size},  bbox_x1: {pos[0]}, bbox_x2: {pos[2]}, bbox_y1: {pos[1]}, bbox_y2: {pos[3]}')\n        return img\n\n    return img.crop((x1,y1,x2,y2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(model_name, image_size):\n    base_model = model_name(weights='imagenet', input_shape=(image_size,image_size,3), include_top=False)\n    \n    model = Sequential()\n    model.add(base_model)\n    model.add(GlobalAveragePooling2D())\n    model.add(Dense(2048, activation='relu'))\n    model.add(Dropout(0.5))\n    model.add(Dense(196, activation='softmax'))\n    model.summary()\n\n    optimizer = Adam(lr=learning_rate)\n    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc', f1_m, precision_m, recall_m])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_steps(num_samples, batch_size):\n    if (num_samples % batch_size) > 0 :\n        return (num_samples // batch_size) + 1\n    else :\n        return num_samples // batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 혹 다른 데이터 셋 추가(Pretrained Model Weights)로 인해 PATH가 변경된다면 아래 PATH를 수정\nDATA_PATH = '../input/2019-3rd-ml-month-with-kakr/'\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 이미지 폴더 경로\nTRAIN_IMG_PATH = os.path.join(DATA_PATH, 'train')\nTEST_IMG_PATH = os.path.join(DATA_PATH, 'test')\n\nCROPPED_IMG_PATH = '../cropped'\n\n# CSV 파일 경로\ndf_train = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))\ndf_class = pd.read_csv(os.path.join(DATA_PATH, 'class.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data 누락 체크\nif set(list(df_train.img_file)) == set(os.listdir(TRAIN_IMG_PATH)) :\n    print(\"Train file 누락 없음!\")\nelse : \n    print(\"Train file 누락\")\n\nif set(list(df_test.img_file)) == set(os.listdir(TEST_IMG_PATH)) :\n    print(\"Test file 누락 없음!\")\nelse : \n    print(\"Test file 누락\")\n    \n# Data 갯수\nprint(\"Number of Train Data : {}\".format(df_train.shape[0]))\nprint(\"Number of Test Data : {}\".format(df_test.shape[0]))\n\nprint(\"타겟 클래스 총 갯수 : {}\".format(df_class.shape[0]))\nprint(\"Train Data의 타겟 종류 갯수 : {}\".format(df_train['class'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Preprocessing - Cropping**"},{"metadata":{"trusted":true},"cell_type":"code","source":"if (os.path.isdir(CROPPED_IMG_PATH) == False):\n    os.mkdir(CROPPED_IMG_PATH)\n    \nfor i, row in df_train.iterrows():\n    cropped = crop_boxing_img(row['img_file'])\n    cropped.save(os.path.join(CROPPED_IMG_PATH, row['img_file']))\n\nfor i, row in df_test.iterrows():\n    cropped = crop_boxing_img(row['img_file'])\n    cropped.save(os.path.join(CROPPED_IMG_PATH, row['img_file']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sometimes = lambda aug: iaa.Sometimes(0.5, aug)\nseq = iaa.Sequential(\n        [\n            # apply the following augmenters to most images\n            iaa.Fliplr(0.5), # horizontally flip 50% of all images\n            #iaa.Flipud(0.2), # vertically flip 20% of all images\n            # crop images by -5% to 10% of their height/width\n            sometimes(iaa.CropAndPad(\n                percent=(-0.05, 0.1),\n                pad_mode=ia.ALL,\n                pad_cval=(0, 255)\n            )),\n            sometimes(iaa.Affine(\n                scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)}, # scale images to 80-120% of their size, individually per axis\n                translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)}, # translate by -20 to +20 percent (per axis)\n                rotate=(-45, 45), # rotate by -45 to +45 degrees\n                shear=(-16, 16), # shear by -16 to +16 degrees\n                order=[0, 1], # use nearest neighbour or bilinear interpolation (fast)\n                cval=(0, 255), # if mode is constant, use a cval between 0 and 255\n                mode=ia.ALL # use any of scikit-image's warping modes (see 2nd image from the top for examples)\n            )),\n            # execute 0 to 5 of the following (less important) augmenters per image\n            # don't execute all of them, as that would often be way too strong\n            iaa.SomeOf((0, 5),\n                [\n                    sometimes(iaa.Superpixels(p_replace=(0, 1.0), n_segments=(20, 200))), # convert images into their superpixel representation\n                    iaa.OneOf([\n                        iaa.GaussianBlur((0, 3.0)), # blur images with a sigma between 0 and 3.0\n                        iaa.AverageBlur(k=(2, 7)), # blur image using local means with kernel sizes between 2 and 7\n                        iaa.MedianBlur(k=(3, 11)), # blur image using local medians with kernel sizes between 2 and 7\n                    ]),\n                    iaa.Sharpen(alpha=(0, 1.0), lightness=(0.75, 1.5)), # sharpen images\n                    iaa.Emboss(alpha=(0, 1.0), strength=(0, 2.0)), # emboss images\n                    # search either for all edges or for directed edges,\n                    # blend the result with the original image using a blobby mask\n                    iaa.SimplexNoiseAlpha(iaa.OneOf([\n                        iaa.EdgeDetect(alpha=(0.5, 1.0)),\n                        iaa.DirectedEdgeDetect(alpha=(0.5, 1.0), direction=(0.0, 1.0)),\n                    ])),\n                    iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5), # add gaussian noise to images\n                    iaa.OneOf([\n                        iaa.Dropout((0.01, 0.1), per_channel=0.5), # randomly remove up to 10% of the pixels\n                        iaa.CoarseDropout((0.03, 0.15), size_percent=(0.02, 0.05), per_channel=0.2),\n                    ]),\n                    iaa.Invert(0.05, per_channel=True), # invert color channels\n                    iaa.Add((-10, 10), per_channel=0.5), # change brightness of images (by -10 to 10 of original value)\n                    iaa.AddToHueAndSaturation((-20, 20)), # change hue and saturation\n                    # either change the brightness of the whole image (sometimes\n                    # per channel) or change the brightness of subareas\n                    iaa.OneOf([\n                        iaa.Multiply((0.5, 1.5), per_channel=0.5),\n                        iaa.FrequencyNoiseAlpha(\n                            exponent=(-4, 0),\n                            first=iaa.Multiply((0.5, 1.5), per_channel=True),\n                            second=iaa.ContrastNormalization((0.5, 2.0))\n                        )\n                    ]),\n                    iaa.ContrastNormalization((0.5, 2.0), per_channel=0.5), # improve or worsen the contrast\n                    iaa.Grayscale(alpha=(0.0, 1.0)),\n                    sometimes(iaa.ElasticTransformation(alpha=(0.5, 3.5), sigma=0.25)), # move pixels locally around (with random strengths)\n                    sometimes(iaa.PiecewiseAffine(scale=(0.01, 0.05))), # sometimes move parts of the image around\n                    sometimes(iaa.PerspectiveTransform(scale=(0.01, 0.1)))\n                ],\n                random_order=True\n            )\n        ],\n        random_order=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class My_Generator(Sequence):\n\n    def __init__(self, image_filenames, labels,\n                 batch_size, is_train=True,\n                 mix=False, augment=False):\n        self.image_filenames, self.labels = image_filenames, labels\n        self.batch_size = batch_size\n        self.is_train = is_train\n        self.is_augment = augment\n        if(self.is_train):\n            self.on_epoch_end()\n        self.is_mix = mix\n\n    def __len__(self):\n        return int(np.ceil(len(self.image_filenames) / float(self.batch_size)))\n\n    def __getitem__(self, idx):\n        batch_x = self.image_filenames[idx * self.batch_size:(idx + 1) * self.batch_size]\n        batch_y = self.labels[idx * self.batch_size:(idx + 1) * self.batch_size]\n\n        if(self.is_train):\n            return self.train_generate(batch_x, batch_y)\n        return self.valid_generate(batch_x, batch_y)\n\n    def on_epoch_end(self):\n        if(self.is_train):\n            self.image_filenames, self.labels = shuffle(self.image_filenames, self.labels)\n        else:\n            pass\n    \n    def mix_up(self, x, y):\n        lam = np.random.beta(0.2, 0.4)\n        ori_index = np.arange(int(len(x)))\n        index_array = np.arange(int(len(x)))\n        np.random.shuffle(index_array)        \n        \n        mixed_x = lam * x[ori_index] + (1 - lam) * x[index_array]\n        mixed_y = lam * y[ori_index] + (1 - lam) * y[index_array]\n        \n        return mixed_x, mixed_y\n\n    def train_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            path = os.path.join(CROPPED_IMG_PATH,sample)\n            img = cv2.imread(path)\n            img = cv2.resize(img, (img_size, img_size))\n            if(self.is_augment):\n                img = seq.augment_image(img)\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        if(self.is_mix):\n            batch_images, batch_y = self.mix_up(batch_images, batch_y)\n        return batch_images, batch_y\n\n    def valid_generate(self, batch_x, batch_y):\n        batch_images = []\n        for (sample, label) in zip(batch_x, batch_y):\n            path = os.path.join(CROPPED_IMG_PATH,sample)\n            img = cv2.imread(path)\n            img = cv2.resize(img, (img_size, img_size))\n            batch_images.append(img)\n        batch_images = np.array(batch_images, np.float32) / 255\n        batch_y = np.array(batch_y, np.float32)\n        return batch_images, batch_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"class\"] = df_train[\"class\"].astype('str')\n\ndf_train = df_train[['img_file', 'class']]\ndf_test = df_test[['img_file']]\n\nits = np.arange(df_train.shape[0])\ntrain_idx, val_idx = train_test_split(its, train_size = 0.8, random_state=42, stratify=df_train[\"class\"])\n\nX_train = df_train.iloc[train_idx, :]\nX_val = df_train.iloc[val_idx, :]\n\nnb_train_samples = len(X_train)\nnb_validation_samples = len(X_val)\n    \ntrain_y = to_categorical(pd.to_numeric(X_train['class'], errors='coerce')-1, num_classes=196)\nvalid_y = to_categorical(pd.to_numeric(X_val['class'], errors='coerce')-1, num_classes=196) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training**"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = './'\n        \ntrain_generator = My_Generator(X_train['img_file'], train_y, batch_size, is_train=True, mix=False, augment=True)\nvalid_generator = My_Generator(X_val['img_file'], valid_y, batch_size, is_train=False)\n\nmodel_name = model_path + net_name + '.hdf5'\n\nmodel = get_model(application, img_size)\n\ntry:\n    model.load_weights(model_name)\nexcept:\n    pass\n\nes = EarlyStopping(monitor='val_f1_m', min_delta=0, patience=patience, verbose=1, mode='max')\nrr = ReduceLROnPlateau(monitor = 'val_f1_m', factor = 0.5, patience = patience/2, min_lr=min_learning_rate, verbose=1, mode='max')\ncl = CSVLogger(filename='../working/training_log.csv', separator=',', append=True)\nmc = ModelCheckpoint(filepath=model_name, monitor='val_f1_m', verbose=1, save_best_only=True, mode='max')\n\ncallbackList = [cl, es, rr, mc]\n\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(lr=learning_rate), metrics=['acc', f1_m, precision_m, recall_m])\n\nhistory = model.fit_generator(\n    train_generator,\n    steps_per_epoch=get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data=valid_generator,\n    validation_steps=get_steps(nb_validation_samples, batch_size),\n    callbacks = callbackList\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Inference**"},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))\n\nmodel_name = model_path + net_name + '.hdf5'\nmodel = get_model(application, img_size)\nmodel.load_weights(model_name)\n\nscore_predict = []\nfor i, img_name in tqdm(enumerate(submit['img_file'])):\n    path = os.path.join(CROPPED_IMG_PATH, img_name)\n    image = cv2.imread(path)\n    image = cv2.resize(image, (img_size, img_size))\n    X = np.array((image[np.newaxis])/255)\n    score_predict.append(model.predict(X).ravel())\n    \nlabel_predict = np.argmax(score_predict, axis=1)\nsubmit['class'] = label_predict+1\nsubmit.to_csv('submission.csv', index=False)\nsubmit.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}