{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport feather\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas()\n# from tqdm import tqdm\n\n# pd.options.display.max_rows = 999\n# pd.options.display.max_columns = 999\nimport glob\ndef get_path(str, first=True, parent_dir='../input/**/'):\n    res_li = glob.glob(parent_dir+str)\n    return res_li[0] if first else res_li","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"DATA_DIR = '../input/dogs-vs-cats-redux-kernels-edition/'\nevals = pd.read_csv('../input/dvc-prepare-evalset/evals.csv')\nevals.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"052c6b0c7448a11076309913be8f2116d9e3ac3c"},"cell_type":"code","source":"H, W, C = 224, 224, 3 #pretrained model requires at least 197\nbatch_size = 32\neval_batch_size = batch_size * 4\nbatch_size = eval_batch_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67431efa32ae1ebe3be5e4e80b6e5d0e005bd0fe"},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.applications.vgg16 import preprocess_input\n\ntrain_gen = ImageDataGenerator(\n    #rotation_range=20,\n    #width_shift_range=0.2,\n    #height_shift_range=0.2,\n    #channel_shift_range=0.2,\n    #vertical_flip=True,\n    #shear_range=0.2,\n    #zoom_range=0.2,\n    #horizontal_flip=True,\n    #rescale=1./255,#!!!!!\n    preprocessing_function=preprocess_input\n)\ntest_gen = ImageDataGenerator(\n    #rescale=1./255,#!!!!!\n    preprocessing_function=preprocess_input\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6578577aa7d1fc4e10b29220b4088c7325e1a7b"},"cell_type":"code","source":"train_flow = train_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=batch_size,\n    shuffle=True,\n)\nvalid_flow = test_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=eval_batch_size,\n    shuffle=False,\n)\ntest_flow = test_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=eval_batch_size,\n    shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4c8686e961326bc61fe1592c73fa7e0216fcc26a"},"cell_type":"code","source":"def set_data_flow(flow, eval_mode, shuffle=True, valid_fold=0, n_valid=128*8, evals=evals):\n    flow.class_indices = {'dog': 0, 'cat': 1}\n    if eval_mode=='train':\n        flow.directory = DATA_DIR+'train'\n        mask = (evals['is_test']==0) & (evals['eval_set']!=valid_fold)\n    elif eval_mode=='valid':\n        shuffle = False\n        flow.directory = DATA_DIR+'train'\n        mask = (evals['is_test']==0) & (evals['eval_set']==valid_fold)\n    elif eval_mode=='test':\n        shuffle = False\n        flow.directory = DATA_DIR+'test'\n        mask = (evals['is_test']==1)\n    flow.samples = len(evals.loc[mask, 'target'].values) if eval_mode!='valid' else n_valid\n    flow.n = len(evals.loc[mask, 'target'].values) if eval_mode!='valid' else n_valid\n    filenames_arr = evals.loc[mask, 'img_id'].apply(lambda x: x+'.jpg').values\n    target_arr = evals.loc[mask, 'target'].values\n    if eval_mode=='valid':\n        filenames_arr = filenames_arr[:n_valid]\n        target_arr = target_arr[:n_valid]\n    if shuffle:\n        indexes = np.arange(flow.samples)\n        np.random.permutatione(indexes)\n        filenames_arr = filenames_arr[indexes]\n        target_arr = target_arr[indexes]\n    flow.filenames = filenames_arr.tolist()\n    flow.classes = target_arr\n    flow.class_mode = 'binary'\n    flow.num_classes = len(np.unique(target_arr))\n    print(f'Found {flow.n} images belonging to {flow.num_classes} classes.')\n    return flow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bca91eb0c1c9ed028b411bc9d5b7bfa3ebb94cab"},"cell_type":"code","source":"train_flow = set_data_flow(train_flow, 'valid', valid_fold=0)\nvalid_flow = set_data_flow(valid_flow, 'valid', valid_fold=1)\ntest_flow = set_data_flow(test_flow, 'test', valid_fold=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"87633e3d1be62879f08484db548858b93fcd0c49"},"cell_type":"code","source":"# MODEL_NAME = f'resnet50_weights_tf_dim_ordering_tf_kernels_notop'\n# MODEL_PATH = f'../input/keras-pretrained-models/{MODEL_NAME}.h5'\nMODEL_NAME = f'vgg16_weights_tf_dim_ordering_tf_kernels_notop'\nMODEL_PATH = f'../input/keras-pretrained-models/{MODEL_NAME}.h5'\nfrom keras.applications.vgg16 import VGG16","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"31daeaf02f9d51b41441e6a0ae5ff4914dc4938e"},"cell_type":"code","source":"def get_pretrained_model(weight_path=MODEL_PATH, trainable=False):\n    input_shape = (H, W, C)\n    #base_model = ResNet50(weights=None, include_top=False, input_shape=input_shape)\n    base_model = VGG16(weights=None, include_top=False, input_shape=input_shape)\n    base_model.load_weights(weight_path)\n    for l in base_model.layers:\n        l.trainable = trainable\n    return base_model\n\nencoder = get_pretrained_model(weight_path=MODEL_PATH, trainable=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c7921dc83d5bbae6772a5ce11f1a6d322e05780"},"cell_type":"code","source":"for bx, by in valid_flow: break\ntmp = encoder.predict(bx)\ntmp.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"565392895c3ddec5374d5b6f0bb085609be07f88"},"cell_type":"code","source":"train_steps = int(np.ceil(train_flow.n / batch_size))\nvalid_steps = int(np.ceil(valid_flow.n / eval_batch_size))\ntest_steps = int(np.ceil(test_flow.n / eval_batch_size))\nprint(f'train {train_steps} steps')\nprint(f'valid {valid_steps} steps')\nprint(f'test {test_steps} steps')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c52f11039c29de68ca14f164b378cb0a4ae1497"},"cell_type":"code","source":"%%time\nX_train = []\ny_train = []\nfor i in tqdm(range(train_steps)):\n    bx,by = next(train_flow)\n    X_train.extend(encoder.predict(bx))\n    y_train.extend(by)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa9d7e420ff169d8bc3a4c1cf742f94da351e0b6"},"cell_type":"code","source":"np.stack(X_train, 0).shape, np.stack(y_train).shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb4a629fc76fda9dbc09d4400b7b71b9c5fb89b7"},"cell_type":"code","source":"%%time\nX_valid = []\ny_valid = []\nfor i in tqdm(range(valid_steps)):\n    bx,by = next(valid_flow)\n    X_valid.extend(encoder.predict(bx))\n    y_valid.extend(by)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"815fa1faac56c780a7c3091ead3bf5859aeeca82","collapsed":true},"cell_type":"code","source":"# %%time\n# X_test = []\n# y_test = []\n# for i in tqdm(range(test_steps)):\n#     bx,by = next(test_flow)\n#     X_test.extend(encoder.predict(bx))\n#     y_test.extend(by)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9e3b743862fcf12f9e403947b9bf60ffaeb4327"},"cell_type":"code","source":"def get_stacked_data(X, y):\n    X, y = np.stack(X, 0), np.stack(y)\n    print(X.shape, y.shape)\n    return X, y\nX_train, y_train = get_stacked_data(X_train, y_train)\nX_valid, y_valid = get_stacked_data(X_valid, y_valid)\n# X_test, y_test = get_stacked_data(X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97f8016139bef69930bf4a099acdd0e0c1147250"},"cell_type":"code","source":"def get_reshaped_data(X, y):\n    X, y = X.reshape(X.shape[0], -1), y\n    print(X.shape, y.shape)\n    return X, y\nX_train, y_train = get_reshaped_data(X_train, y_train)\nX_valid, y_valid = get_reshaped_data(X_valid, y_valid)\n# X_test, y_test = get_reshaped_data(X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28b31028dfa90fd4a2f48918cceedd5e6565e6b9","collapsed":true},"cell_type":"code","source":"from scipy import sparse\ndef save_feature(savename, X, y):\n    #np.save(f'X_{savename}.npy', X) #big\n    sparse.save_npz(f'X_{savename}.npz', sparse.csr_matrix(X), compressed=True)\n    np.save(f'y_{savename}.npy', y)\n    \nsave_feature('train', X_train, y_train)\nsave_feature('valid', X_valid, y_valid)\n# save_feature('test', X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"efaa4bc54fdec63aafba5a04a92c9591dfff4004"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.models import Model\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping\nfrom keras import optimizers, losses, activations, models\nfrom keras.layers import Conv2D, Dense, Input, Flatten, Concatenate, Dropout, Activation\nfrom keras.layers import BatchNormalization, MaxPooling2D, GlobalAveragePooling2D\nfrom keras import applications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6dec59c0841a9c085158f3cd1a2d5f08a1dc998"},"cell_type":"code","source":"n_feature = X_train.shape[1]\nn_feature","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1f688e6499fc81181ffe6db904777578d0a7d2f"},"cell_type":"code","source":"n_final_state = 32\n\ndef get_model(n_final_state, lr=1e-3, decay=1e-8):\n    #input_shape = (H, W, C)\n    input_shape = (n_feature,)\n    \n    input_x = Input(shape=input_shape)\n    \n    d1 = Dense(\n        64, activation='relu'\n    )(input_x)\n    #d1 = Dropout(0.5)(d1)\n    d1 = BatchNormalization()(d1)\n    \n    final_state = Dense(\n        n_final_state, activation='relu', name='final_state'\n    )(d1)\n    \n    x = Dropout(0.5)(final_state)\n    outputs = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=input_x, outputs=outputs)\n    optimizer=optimizers.Adam(lr=lr, decay=decay)\n    model.compile(loss='binary_crossentropy', \n                  optimizer=optimizer,\n                  metrics=['accuracy'])\n    return model\n\nmodel = get_model(n_final_state=n_final_state)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef3f386c701944e1483ff2cb00867c28733f730d"},"cell_type":"code","source":"epochs = 10\n\nprint('BATCH_SIZE: {} EPOCHS: {}'.format(batch_size, epochs))\n\nfile_path='model.h5'\ncheckpoint = ModelCheckpoint(\n    file_path, monitor='val_loss', verbose=1, \n    save_best_only=True, \n    save_weights_only=True,\n    mode='min'\n)\nearly = EarlyStopping(monitor='val_loss', mode='min', patience=30)\ncallbacks_list = [checkpoint, early]\n\nK.set_value(model.optimizer.lr, 0.0005)\n\ngc.collect();\nhistory = model.fit(\n    X_train, y_train,\n    batch_size=batch_size,\n    validation_data=(X_valid, y_valid),\n    epochs=epochs, \n    verbose=1,\n    shuffle=False,\n    callbacks=callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"74e21f4a349bf29c5ba1394aa048e80d5c5713a5"},"cell_type":"code","source":"model.load_weights(file_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"420b1b8050c3195f672c27e600f0dd39ae5930c9"},"cell_type":"code","source":"pred_val = model.predict(X_valid)\npred_val = pred_val.ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f02cd74515656fe5c0e5fc93e076c71d0a27a1"},"cell_type":"code","source":"from sklearn.metrics import log_loss, accuracy_score\nval_loss = log_loss(y_valid, pred_val)\nval_acc = accuracy_score(y_valid, np.round(pred_val))\nprint(f'valid loss: {val_loss}\\t valid accuracy: {val_acc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"960893bdef553227d9bd8b7b26a753d33d0d1575","collapsed":true},"cell_type":"code","source":"np.save('valid_pred.npy', pred_val)\n# np.save('test_pred.npy', pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad1dfbfd1b90687c265de84e602224e633878a5a","collapsed":true},"cell_type":"code","source":"# mask = evals['is_test']==1\n# sub = {\n#     'id': evals.loc[mask, 'img_id'].values.astype('int'),\n#     'label': pred_test,\n# }\n# sub = pd.DataFrame(sub).sort_values(by='id').reset_index(drop=True)\n# sub['label'] = 1 - sub['label']\n\n# subname = f'resnet50ft_{val_loss:.6f}.csv'\n# sub.to_csv(subname, index=False)\n# print(subname, 'saved')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}