{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport gc\nimport os\nimport warnings\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom keras.layers import Input\nfrom keras.models import Model\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications import Xception\nfrom keras.applications.resnet50 import ResNet50, preprocess_input\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\nfrom keras import layers, models, optimizers\nfrom keras import regularizers\n#from keras.regularizers import l2\n\nfrom keras import backend as K\nwarnings.filterwarnings(action='ignore')\n%matplotlib inline\n\n\nimport PIL\nfrom PIL import ImageOps, ImageFilter, ImageDraw\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_PATH = '../input/'\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 이미지 폴더 경로\nTRAIN_IMG_PATH = os.path.join(DATA_PATH, 'train')\nTEST_IMG_PATH = os.path.join(DATA_PATH, 'test')\n\n# CSV 파일 경로\ndf_train = pd.read_csv(os.path.join(DATA_PATH, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(DATA_PATH, 'test.csv'))\ndf_class = pd.read_csv(os.path.join(DATA_PATH, 'class.csv'))\n\nimage_size = 299\nbatch_size = 16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape, df_test.shape, df_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train Data의 class 분포 확인 \nplt.figure(figsize=(12, 6))\nsns.countplot(df_train[\"class\"], order=df_train[\"class\"].value_counts(ascending=True).index)\nplt.title(\"Number of data per each class\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Cropped Image Dataset\n\n제공된 바인딩 박스 x1(좌상단), y1(좌상단), x2(우하단), y2(우하단) 정보를 이용해 배경을 잘라낸다."},{"metadata":{"trusted":true},"cell_type":"code","source":"def crop_boxing_img(img_name, margin=16, size=(image_size, image_size)):\n    if img_name.split('_')[0] == 'train':\n        PATH = TRAIN_IMG_PATH\n        data = df_train\n    else:\n        PATH = TEST_IMG_PATH\n        data = df_test\n\n    img = PIL.Image.open(os.path.join(PATH, img_name))\n    pos = data.loc[data[\"img_file\"] == img_name, ['bbox_x1', 'bbox_y1', 'bbox_x2', 'bbox_y2']].values.reshape(-1)\n\n    width, height = img.size\n    x1 = max(0, pos[0] - margin)\n    y1 = max(0, pos[1] - margin)\n    x2 = min(pos[2] + margin, width)\n    y2 = min(pos[3] + margin, height)\n\n    return img.crop((x1, y1, x2, y2)).resize(size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train : validation data = 9 : 1\nTRAIN_CROPPED_PATH = \"./train_crop\"\nVALID_CROPPED_PATH = '/valid_crop'\nTEST_CROPPED_PATH = \"./test_crop\"\n\nnb_train_sample = df_train.shape[0] * 0.9\nnb_validation_sample = df_train.shape[0] - nb_train_sample\nnb_test_sample = df_test.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Make Directory \nif (os.path.isdir(TRAIN_CROPPED_PATH) == False):\n    os.mkdir(TRAIN_CROPPED_PATH)\n\nif (os.path.isdir(VALID_CROPPED_PATH) == False):\n    os.mkdir(VALID_CROPPED_PATH)\n\nif (os.path.isdir(TEST_CROPPED_PATH) == False):\n    os.mkdir(TEST_CROPPED_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 클래스별 디렉토리를 만들기 위해서 class 형을 문자열로 변경\ndf_train['class'] = df_train['class'].astype('str')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, row in df_train.iterrows():\n    cropped = crop_boxing_img(row['img_file'])\n    \n    if ( i < nb_train_sample):\n        class_path = os.path.join(TRAIN_CROPPED_PATH, df_train['class'][i])\n        if(os.path.isdir(class_path) == False):\n            os.mkdir(class_path)\n\n        cropped.save(os.path.join(class_path, row['img_file']))\n    else:\n        class_path = os.path.join(VALID_CROPPED_PATH, df_train['class'][i])\n        if(os.path.isdir(class_path) == False):\n            os.mkdir(class_path)\n\n        cropped.save(os.path.join(class_path, row['img_file']))\n\nfor i, row in df_test.iterrows():\n    cropped = crop_boxing_img(row['img_file'])\n    cropped.save(os.path.join(TEST_CROPPED_PATH, row['img_file']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generator\n\n이미지 학습간에는 거의 필수작업, 배치 사이즈 단위 만큼만 파일을 불러와서 학습이 끝나면 다시 불러와 학습을 반복하기에 메모리를 절약할 수 있다.\n\n\n\n#### keras의 ImageDataGenerator를 이용하여 데이터 증식을 합니다.\n\n - rotation_range : 랜덤하게 사진을 회전시킬 각도 범위\n - width_shift_range : 사진 수평이동 비율\n - height_shift_range : 사진 수직이동 비율\n - horizontal_flip : 사진 수평으로 뒤집기\n - zoom_range : 랜덤하게 사진 확대\n - fill_mode : 사진을 회전/이동한후에 새롭게 생성해야할 픽셀을 채울 방법"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20.,\n    width_shift_range=0.5,\n    height_shift_range=0.5,\n    shear_range = 0.1,\n    horizontal_flip=True,\n    vertical_flip=False,\n    zoom_range=0.3,\n#    fill_mode='nearest'\n    fill_mode='reflect'\n    )\n\nvalid_datagen = ImageDataGenerator(rescale=1./255)\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\n\n# Make Generator\ntrain_generator = train_datagen.flow_from_directory(\n    TRAIN_CROPPED_PATH,\n    target_size=(image_size, image_size),\n    batch_size=batch_size,\n    class_mode='categorical',\n    seed=2019,\n    color_mode='rgb'\n)\n\nvalidation_generator = valid_datagen.flow_from_directory(\n    VALID_CROPPED_PATH,\n    target_size=(image_size,image_size),\n    batch_size=batch_size,\n    class_mode='categorical',\n    seed=2019,\n    color_mode='rgb'\n)\n\ntest_generator = test_datagen.flow_from_dataframe(\n    dataframe=df_test,\n    directory=TEST_CROPPED_PATH,\n    x_col='img_file',\n    y_col=None,\n    target_size= (image_size,image_size),\n    color_mode='rgb',\n    class_mode=None,\n    batch_size=batch_size,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model 생성"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = '../model/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_model(application):\n    \n    input_layer = Input(shape=(image_size,image_size, 3))\n    base_model = application(weights='imagenet', include_top=False)(input_layer)\n\n    \n    x = layers.GlobalAveragePooling2D()(base_model)\n    x = layers.Dense(1024, activation='relu',\n                     bias_initializer='zeros',\n                     kernel_initializer='he_normal',\n                     kernel_regularizer=regularizers.L1L2(l1=0.001, l2=0.001))(x)\n    #model.add(layers.Dense(1024, activation='relu', bias_initializer='zeros', kernel_initializer='he_normal', kernel_regularizer=l2(0.001)))\n    \n    x = layers.BatchNormalization()(x)\n    x = layers.Dropout(0.5)(x)\n\n    output_layer = layers.Dense(196, activation='softmax')(x)    \n    model = Model(input_layer, output_layer)\n    model.summary()\n\n    #optimizer = optimizers.SGD(lr=1e-4, decay=1e-6, momentum=0.9, nesterov=True)\n    #optimizer = optimizers.RMSprop(lr=0.0001)\n    optimizer = optimizers.nadam(lr=0.0001)\n    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['acc'])\n\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xception_best_model = 'best_Xception_model.hdf5'\nResNet50_best_model = 'best_ResNet50_model.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Xmodel = get_model(Xception)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Rmodel = get_model(ResNet50)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 훈련데이터 저장\n\n훈련하면서 검증데이터에 대해서 val_loss가 가장 작은 경우에 대해서 weight 저장. \n이 파일명을 잘 관리해 두었다가 나중에 앙상블에 이용할 예정임. best_model.hd5로 저장을 하다가 훈련이 끝나면 bestmodel(사전훈련커브넷).hd5로 저장."},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists(model_path):\n    os.mkdir(model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"patient = 3\ncallbacks1 = [\n    EarlyStopping(monitor='val_loss', patience=patient, mode='min', verbose=1),\n    \n    # val_loss 이 향상되지 않을 때 학습률을 작게 한다. (factor = 0.5 : 콜백이 호출되면 학습률을 50배로 줄임.)\n    #ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = patient / 2, min_lr=0.00001, verbose=1, mode='min'),\n    ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 1, min_lr=0.00001, verbose=1, mode='min'),\n    \n    # 모델을 저장하고 복원하기 위한 콜백 함수. (훈련하는 동안 어떤 지점에서 모델의 현재 가중치를 저장한다.)\n    ModelCheckpoint(filepath=model_path+Xception_best_model, monitor='val_loss', verbose=1, save_best_only=True, mode='min'),\n    ]\n\ncallbacks2 = [\n    EarlyStopping(monitor='val_loss', patience=patient, mode='min', verbose=1),\n    \n    # val_loss 이 향상되지 않을 때 학습률을 작게 한다. (factor = 0.5 : 콜백이 호출되면 학습률을 50배로 줄임.)\n    #ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = patient / 2, min_lr=0.00001, verbose=1, mode='min'),\n    ReduceLROnPlateau(monitor = 'val_loss', factor = 0.1, patience = 1, min_lr=0.00001, verbose=1, mode='min'),\n    \n    # 모델을 저장하고 복원하기 위한 콜백 함수. (훈련하는 동안 어떤 지점에서 모델의 현재 가중치를 저장한다.)\n    ModelCheckpoint(filepath=model_path+ResNet50_best_model, monitor='val_loss', verbose=1, save_best_only=True, mode='min'),\n    ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_steps(num_samples, batch_size):\n    if (num_samples % batch_size) > 0 :\n        return (num_samples // batch_size) + 1\n    else :\n        return num_samples // batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = Xmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=get_steps(nb_train_sample, batch_size) * 2,\n    epochs=100,\n    validation_data=validation_generator,\n    validation_steps=get_steps(nb_validation_sample, batch_size),\n    verbose=1,\n    callbacks = callbacks1\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'bo', label='Training Acc')\nplt.plot(epochs, val_acc, 'b', label='Validation Acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.show()\n\nplt.plot(epochs, loss, 'bo', label='Traing loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Trainging and validation loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = Rmodel.fit_generator(\n    train_generator,\n    steps_per_epoch=get_steps(nb_train_sample, batch_size) * 2,\n    epochs=100,\n    validation_data=validation_generator,\n    validation_steps=get_steps(nb_validation_sample, batch_size),\n    verbose=1,\n    callbacks = callbacks2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'bo', label='Training Acc')\nplt.plot(epochs, val_acc, 'b', label='Validation Acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.show()\n\nplt.plot(epochs, loss, 'bo', label='Traing loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Trainging and validation loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = []\ntta_steps = 10\n\nXmodel.load_weights(model_path+Xception_best_model)\nRmodel.load_weights(model_path+ResNet50_best_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TTA 출처 : https://towardsdatascience.com/test-time-augmentation-tta-and-how-to-perform-it-with-keras-4ac19b67fb4d\npredictions = []\ntest_generator.reset()\n\nfor i in tqdm(range(tta_steps)):\n    Xception_prediction = Xmodel.predict_generator(\n        generator=test_generator,\n        steps = get_steps(nb_test_sample, batch_size),\n        verbose=1\n    )\n    test_generator.reset()\n    predictions.append(Xception_prediction)\n    \nfinal_prediction = predictions[0] + predictions[1] + predictions[2] + predictions[3] + predictions[4] + predictions[5] + predictions[6] + predictions[7] + predictions[8] + predictions[9]\nfinal_prediction /= 10.0   \n\nlabels.append(final_prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = []\ntest_generator.reset()\n\nfor i in tqdm(range(tta_steps)):\n    ResNet50_prediction = Rmodel.predict_generator(\n        generator=test_generator,\n        steps = get_steps(nb_test_sample, batch_size),\n        verbose=1\n    )\n    test_generator.reset()\n    predictions.append(ResNet50_prediction)\n\nfinal_prediction = predictions[0] + predictions[1] + predictions[2] + predictions[3] + predictions[4] + predictions[5] + predictions[6] + predictions[7] + predictions[8] + predictions[9]\nfinal_prediction /= 10.0 \n    \nlabels.append(final_prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_prediction = labels[0] + labels[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_prediction /= 2.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"predicted_class_indices=np.argmax(final_prediction, axis=1)\n\n# Generator class dictionary mapping\nlabels = (train_generator.class_indices)\nlabels = dict((v,k) for k,v in labels.items())\npredictions = [labels[k] for k in predicted_class_indices]\n\nsubmission = pd.read_csv(os.path.join(DATA_PATH, 'sample_submission.csv'))\nsubmission[\"class\"] = predictions\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf *_crop","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}