{"cells":[{"metadata":{},"cell_type":"markdown","source":"# <center>3rd ML Month - Compare optimizer of efficientNet </center>"},{"metadata":{},"cell_type":"markdown","source":"# Reference\n- https://shaoanlu.wordpress.com/2017/05/29/sgd-all-which-one-is-the-best-optimizer-dogs-vs-cats-toy-experiment/\n- 커널 시간제한으로 EarlyStopping 설정, 동일 epoch 비교 X"},{"metadata":{},"cell_type":"markdown","source":"# Package"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport warnings\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import f1_score\nfrom keras import backend as K\n# for문 시간계산 lib\nfrom tqdm import tqdm_notebook\n# 교차검증 lib\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split\n#모델 lib\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential, Model\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nfrom keras.layers import Dense, Dropout, Flatten, Activation, Conv2D, GlobalAveragePooling2D\nfrom keras import layers\nfrom keras.optimizers import Adam,RMSprop,SGD,Nadam\n#경고메세지 무시\nimport warnings\nwarnings.filterwarnings(action='ignore')\n#input 하위 디렉토리 폴터\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#efficientnet download\n!pip install git+https://github.com/qubvel/efficientnet\nfrom efficientnet import EfficientNetB3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# File Directory Setting"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#crop data directory\nDATA_PATH = '../input/car-crop'\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#original data directory\nDATA_PATH2 = '../input/2019-3rd-ml-month-with-kakr'\nos.listdir(DATA_PATH2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 이미지 폴더 경로\nTRAIN_IMG_PATH = os.path.join(DATA_PATH, 'train')\nTEST_IMG_PATH = os.path.join(DATA_PATH, 'test')\n\n# CSV 파일 경로\ndf_train = pd.read_csv(os.path.join(DATA_PATH2, 'train.csv'))\ndf_test = pd.read_csv(os.path.join(DATA_PATH2, 'test.csv'))\ndf_class = pd.read_csv(os.path.join(DATA_PATH2, 'class.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# train/test data Split"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"class\"] = df_train[\"class\"].astype('str')\n\ndf_train = df_train[['img_file', 'class']]\ndf_test = df_test[['img_file']]\n\nits = np.arange(df_train.shape[0])\ntrain_idx, val_idx = train_test_split(its, train_size = 0.8, random_state=42)\n\nX_train = df_train.iloc[train_idx, :]\nX_val = df_train.iloc[val_idx, :]\n\nprint(X_train.shape)\nprint(X_val.shape)\nprint(df_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Parameter"},{"metadata":{"trusted":true},"cell_type":"code","source":"def recall_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\ndef precision_m(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Parameter\nimg_size = (299, 299)\nimage_size = 299\nnb_train_samples = len(X_train)\nnb_validation_samples = len(X_val)\nnb_test_samples = len(df_test)\nepochs = 20\nbatch_size = 32\n\n# Define Generator config\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    horizontal_flip=True,\n    vertical_flip=False,\n    zoom_range=0.2,\n    fill_mode='nearest')\nval_datagen = ImageDataGenerator(rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#generator\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe=X_train, \n    directory='../input/car-crop/train_crop',\n    x_col = 'img_file',\n    y_col = 'class',\n    target_size = img_size,\n    color_mode='rgb',\n    class_mode='categorical',\n    batch_size=batch_size,\n    seed=42\n)\n\nvalidation_generator = val_datagen.flow_from_dataframe(\n    dataframe=X_val, \n    directory='../input/car-crop/train_crop',\n    x_col = 'img_file',\n    y_col = 'class',\n    target_size = img_size,\n    color_mode='rgb',\n    class_mode='categorical',\n    batch_size=batch_size,\n    shuffle=False,\n    seed=42\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_steps(num_samples, batch_size):\n    if (num_samples % batch_size) > 0 :\n        return (num_samples // batch_size) + 1\n    else :\n        return num_samples // batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\n\n#model path\nMODEL_SAVE_FOLDER_PATH = './model/'\nif not os.path.exists(MODEL_SAVE_FOLDER_PATH):\n    os.mkdir(MODEL_SAVE_FOLDER_PATH)\n\nmodel_path = MODEL_SAVE_FOLDER_PATH + '{epoch:02d}-{val_loss:.4f}.hdf5'\n\npatient = 2\ncallbacks_list = [\n    EarlyStopping(\n        # 모델의 검증 정확도 모니터링\n        monitor='val_loss', \n        # patient(정수)보다 정확도가 향상되지 않으면 훈련 종료\n        patience=patient, \n        # 검증에 대해 판단하기 위한 기준, val_loss경우 감소되는 것이므로 min\n        mode='min', \n        #얼마나 자세하게 정보를 나타낼것인가.\n        verbose=1\n                          \n    ),\n    ReduceLROnPlateau(\n        monitor = 'val_loss', \n        #콜백 호출시 학습률(lr)을 절반으로 줄임\n        factor = 0.5, \n        #위와 동일\n        patience = patient / 2, \n        #최소학습률\n        min_lr=0.00001,\n        verbose=1,\n        mode='min'\n    ) ]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model\ndef get_model():\n    EfficientNet_model = base_model = EfficientNetB3(weights='imagenet', include_top=False, \n                                                     input_shape=(299, 299, 3))\n\n\n    model = Sequential()\n    model.add(EfficientNet_model)\n    model.add(layers.GlobalAveragePooling2D())\n    model.add(layers.Dense(2048, activation='relu'))\n    model.add(layers.Dropout(0.25))\n    model.add(layers.Dense(196, activation='softmax'))\n    #model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 1: RMSprop"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_rmsprop = get_model()\nmodel_rmsprop.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc',f1_m])\nhist_rmsprop = model_rmsprop.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 2: Adam"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_adam = get_model()\nmodel_adam.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['acc',f1_m])\nhist_adam = model_adam.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 3: Nadam"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_nadam = get_model()\nmodel_nadam.compile(loss='categorical_crossentropy', optimizer=Nadam(), metrics=['acc',f1_m])\nhist_nadam = model_nadam.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 4: SGD"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_sgd = get_model()\nmodel_sgd.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['acc',f1_m])\nhist_sgd = model_sgd.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 5: SGD + Nesterov"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_sgdnes = get_model()\nmodel_sgdnes.compile(loss='categorical_crossentropy', optimizer=SGD(nesterov=True), metrics=['acc',f1_m])\nhist_sgdnes = model_sgdnes.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 6: SGD with momentum=0.9"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_sgdmo = get_model()\nmodel_sgdmo.compile(loss='categorical_crossentropy', optimizer=SGD(momentum=0.9), metrics=['acc',f1_m])\nhist_sgdmo = model_sgdmo.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Optimizer 7: SGD + Nesterov with momentum=0.9"},{"metadata":{"trusted":true},"cell_type":"code","source":"#compile\nmodel_sgdmones = get_model()\nmodel_sgdmones.compile(loss='categorical_crossentropy', optimizer=SGD(momentum=0.9, nesterov=True), metrics=['acc',f1_m])\nhist_sgdmones  = model_sgdmones.fit_generator(\n    train_generator,\n    steps_per_epoch = get_steps(nb_train_samples, batch_size),\n    epochs=epochs,\n    validation_data = validation_generator,\n    validation_steps = get_steps(nb_validation_samples, batch_size),\n    callbacks = callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# acc / loss Plot"},{"metadata":{},"cell_type":"markdown","source":"### train acc"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['acc'])  \nplt.plot(hist_adam.history['acc'])  \nplt.plot(hist_nadam.history['acc']) \nplt.plot(hist_sgd.history['acc']) \nplt.plot(hist_sgdnes.history['acc']) \nplt.plot(hist_sgdmo.history['acc'])\nplt.plot(hist_sgdmones.history['acc'])\nplt.title('train. accuracy')  \nplt.ylabel('accuracy')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='lower right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['loss'])  \nplt.plot(hist_adam.history['loss'])  \nplt.plot(hist_nadam.history['loss']) \nplt.plot(hist_sgd.history['loss']) \nplt.plot(hist_sgdnes.history['loss']) \nplt.plot(hist_sgdmo.history['loss'])\nplt.plot(hist_sgdmones.history['loss'])\nplt.title('train. loss')  \nplt.ylabel('loss')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='upper right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### valid acc"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['val_acc'])\nplt.plot(hist_adam.history['val_acc'])\nplt.plot(hist_nadam.history['val_acc'])\nplt.plot(hist_sgd.history['val_acc'])\nplt.plot(hist_sgdnes.history['val_acc'])\nplt.plot(hist_sgdmo.history['val_acc'])\nplt.plot(hist_sgdmones.history['val_acc'])\n\nplt.title('valid. accuracy')  \nplt.ylabel('accuracy')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='lower right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### valid loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['val_loss'])  \nplt.plot(hist_adam.history['val_loss'])  \nplt.plot(hist_nadam.history['val_loss']) \nplt.plot(hist_sgd.history['val_loss']) \nplt.plot(hist_sgdnes.history['val_loss']) \nplt.plot(hist_sgdmo.history['val_loss'])\nplt.plot(hist_sgdmones.history['val_loss'])\nplt.title('valid. loss')  \nplt.ylabel('loss')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='upper right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train f1 score"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['f1_m'])  \nplt.plot(hist_adam.history['f1_m'])  \nplt.plot(hist_nadam.history['f1_m']) \nplt.plot(hist_sgd.history['f1_m']) \nplt.plot(hist_sgdnes.history['f1_m']) \nplt.plot(hist_sgdmo.history['f1_m'])\nplt.plot(hist_sgdmones.history['f1_m'])\nplt.title('train. f1_score')  \nplt.ylabel('f1_score')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='upper right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### valid f1 score"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10, 6))  \n\nplt.plot(hist_rmsprop.history['val_f1_m'])  \nplt.plot(hist_adam.history['val_f1_m'])  \nplt.plot(hist_nadam.history['val_f1_m']) \nplt.plot(hist_sgd.history['val_f1_m']) \nplt.plot(hist_sgdnes.history['val_f1_m']) \nplt.plot(hist_sgdmo.history['val_f1_m'])\nplt.plot(hist_sgdmones.history['val_f1_m'])\nplt.title('valid. f1_score')  \nplt.ylabel('f1_score')  \nplt.xlabel('epoch')  \nplt.legend(['rmsprop', 'adam', 'nadam', 'sgd', 'sgd+nesterov', 'sgd+momentum', 'sgd+nesterov+momentum'], loc='upper right')  \n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 결론\n-  'sgd', 'sgd+nesterov' 는 너무 늦게 수렴하여 comp에 적절한 optimizer는 아닌것 같습니다.\n-  'rmsprop', 'adam'이 비교적으로 빠른 시간안에 높은 acc에 도달합니다."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}