{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\nimport os\nimport gc\nimport time\nimport pickle\nimport feather\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm._tqdm_notebook import tqdm_notebook as tqdm\ntqdm.pandas()\n# from tqdm import tqdm\n\n# pd.options.display.max_rows = 999\n# pd.options.display.max_columns = 999\nimport glob\ndef get_path(str, first=True, parent_dir='../input/**/'):\n    res_li = glob.glob(parent_dir+str)\n    return res_li[0] if first else res_li","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"DATA_DIR = '../input/dogs-vs-cats-redux-kernels-edition/'\nevals = pd.read_csv('../input/dvc-prepare-evalset/evals.csv')\nevals.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"052c6b0c7448a11076309913be8f2116d9e3ac3c"},"cell_type":"code","source":"H, W, C = 150, 150, 3\nbatch_size = 32\neval_batch_size = batch_size * 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67431efa32ae1ebe3be5e4e80b6e5d0e005bd0fe","collapsed":true},"cell_type":"code","source":"import tensorflow as tf\nimport keras\nfrom keras.preprocessing.image import ImageDataGenerator\n\ntrain_gen = ImageDataGenerator(\n    rotation_range=20,\n    #width_shift_range=0.2,\n    #height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    #channel_shift_range=0.2,\n    horizontal_flip=True,\n    #vertical_flip=True,\n    #rescale=1./255,#!!!!NO!\n    preprocessing_function=lambda x:(x-x.mean())/x.std()\n)\ntest_gen = ImageDataGenerator(\n    #rescale=1./255,\n    preprocessing_function=lambda x:(x-x.mean())/x.std()\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6578577aa7d1fc4e10b29220b4088c7325e1a7b","collapsed":true},"cell_type":"code","source":"train_flow = train_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=batch_size,\n    shuffle=True,\n)\nvalid_flow = test_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=eval_batch_size,\n    shuffle=False,\n)\ntest_flow = test_gen.flow_from_directory(\n    './', # Empty dir\n    class_mode=None, \n    target_size=(H, W),\n    batch_size=eval_batch_size,\n    shuffle=False,\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4c8686e961326bc61fe1592c73fa7e0216fcc26a"},"cell_type":"code","source":"def set_data_flow(flow, eval_mode, shuffle=True, valid_fold=0, n_valid=128*16, evals=evals):\n    flow.class_indices = {'dog': 0, 'cat': 1}\n    if eval_mode=='train':\n        flow.directory = DATA_DIR+'train'\n        mask = (evals['is_test']==0) & (evals['eval_set']!=valid_fold)\n    elif eval_mode=='valid':\n        shuffle = False\n        flow.directory = DATA_DIR+'train'\n        mask = (evals['is_test']==0) & (evals['eval_set']==valid_fold)\n    elif eval_mode=='test':\n        shuffle = False\n        flow.directory = DATA_DIR+'test'\n        mask = (evals['is_test']==1)\n    flow.samples = len(evals.loc[mask, 'target'].values) if eval_mode!='valid' else n_valid\n    flow.n = len(evals.loc[mask, 'target'].values) if eval_mode!='valid' else n_valid\n    filenames_arr = evals.loc[mask, 'img_id'].apply(lambda x: x+'.jpg').values\n    target_arr = evals.loc[mask, 'target'].values\n    if eval_mode=='valid':\n        filenames_arr = filenames_arr[:n_valid]\n        target_arr = target_arr[:n_valid]\n    if shuffle:\n        indexes = np.arange(flow.samples)\n        np.random.permutatione(indexes)\n        filenames_arr = filenames_arr[indexes]\n        target_arr = target_arr[indexes]\n    flow.filenames = filenames_arr.tolist()\n    flow.classes = target_arr\n    flow.class_mode = 'binary'\n    flow.num_classes = len(np.unique(target_arr))\n    print(f'Found {flow.n} images belonging to {flow.num_classes} classes.')\n    return flow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bca91eb0c1c9ed028b411bc9d5b7bfa3ebb94cab","collapsed":true},"cell_type":"code","source":"train_flow = set_data_flow(train_flow, 'valid', valid_fold=0)\nvalid_flow = set_data_flow(valid_flow, 'valid', valid_fold=1)\ntest_flow = set_data_flow(test_flow, 'test', valid_fold=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3798de263fe2cad4e8d11fe6fec161c5a52f8e2d"},"cell_type":"code","source":"import keras.backend as K\nfrom keras.models import Model\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping\nfrom keras import optimizers, losses, activations, models\nfrom keras.layers import Conv2D, Dense, Input, Flatten, Concatenate, Dropout, Activation\nfrom keras.layers import BatchNormalization, MaxPooling2D, GlobalAveragePooling2D\nfrom keras import applications","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"af5cab2e4b56ef25bdb0c0a7e399ae8228e606e5","collapsed":true},"cell_type":"code","source":"n_final_state = 32\n\ndef get_model(n_final_state, lr=1e-3, decay=1e-8):\n    input_shape = (H, W, C)\n    \n    input_x = Input(shape=input_shape)\n    \n    c1 = Conv2D(32, (3, 3))(input_x)\n    c1 = BatchNormalization()(c1)\n    c1 = Activation('relu')(c1)\n    c1 = MaxPooling2D((2, 2))(c1)\n\n    c2 = Conv2D(32, (3, 3))(c1)\n    c2 = BatchNormalization()(c2)\n    c2 = Activation('relu')(c2)\n    c2 = MaxPooling2D((2, 2))(c2)\n    \n    c3 = Conv2D(64, (3, 3))(c2)\n    c3 = BatchNormalization()(c3)\n    c3 = Activation('relu')(c3)\n    c3 = MaxPooling2D((2, 2))(c3)\n    \n    flat = Flatten()(c3)\n    \n    d1 = Dense(\n        64, activation='relu'\n    )(flat)\n    #d1 = Dropout(0.5)(d1)\n    d1 = BatchNormalization()(d1)\n    \n    final_state = Dense(\n        n_final_state, activation='relu', name='final_state'\n    )(d1)\n    \n    x = Dropout(0.5)(final_state)\n    outputs = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=input_x, outputs=outputs)\n    optimizer=optimizers.Adam(lr=lr, decay=decay)\n    model.compile(loss='binary_crossentropy', \n                  optimizer=optimizer,\n                  metrics=['accuracy'])\n    return model\n\nmodel = get_model(n_final_state=n_final_state)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"19a1e9fae25cd5e012d1814db7f720f2375aa504"},"cell_type":"code","source":"train_steps = int(np.ceil(train_flow.n / batch_size))\nvalid_steps = int(np.ceil(valid_flow.n / eval_batch_size))\ntest_steps = int(np.ceil(test_flow.n / eval_batch_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fbb0a70d7cff24f0a718562e18896597913899c","_kg_hide-output":true,"collapsed":true},"cell_type":"code","source":"epochs = 50\n\nprint('BATCH_SIZE: {} EPOCHS: {}'.format(batch_size, epochs))\nprint(f'train {train_steps} steps')\nprint(f'valid {valid_steps} steps')\nprint(f'test {test_steps} steps')\n\nfile_path='model.h5'\ncheckpoint = ModelCheckpoint(\n    file_path, monitor='val_loss', verbose=1, \n    save_best_only=True, \n    save_weights_only=True,\n    mode='min'\n)\nearly = EarlyStopping(monitor='val_loss', mode='min', patience=30)\ncallbacks_list = [checkpoint, early]\n\nK.set_value(model.optimizer.lr, 0.0005)\n\ngc.collect();\nhistory = model.fit_generator(\n    train_flow, \n    steps_per_epoch=train_steps,\n    validation_data=valid_flow,\n    validation_steps=valid_steps,\n    epochs=epochs, \n    verbose=1,\n    callbacks=callbacks_list\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bc882f97f023a58b10563881bd9f3a36530bb23","collapsed":true},"cell_type":"code","source":"eval_res = pd.DataFrame(history.history)\neval_res.to_csv('eval_res.csv', index=False)\nfor c in ['acc', 'loss']:\n    eval_res[[c, f'val_{c}']].plot(figsize=[18, 6]);\n    plt.xlabel('Epoch'); plt.ylabel(c);\n    plt.title(c); plt.grid();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"de6df74b98ff1a0ff8a36ff88a28d87ebbbaadae"},"cell_type":"code","source":"model.load_weights('model.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df1fc9ea912b7e9212eddf66ba63d954f049211d","collapsed":true},"cell_type":"code","source":"# final_state_model = Model(model.inputs, model.get_layer('final_state').output)\n# valid_state = final_state_model.predict_generator(valid_flow, steps=valid_steps, verbose=1)\npred_val = model.predict_generator(valid_flow, steps=valid_steps, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d37dc22f9ba554f3fd68c30061071f21b719cb17","collapsed":true},"cell_type":"code","source":"pred_val.shape, valid_flow.classes.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"420b1b8050c3195f672c27e600f0dd39ae5930c9","collapsed":true},"cell_type":"code","source":"pred_val = pred_val.ravel()\ny_valid =  valid_flow.classes.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46f02cd74515656fe5c0e5fc93e076c71d0a27a1","collapsed":true},"cell_type":"code","source":"from sklearn.metrics import log_loss, accuracy_score\nval_loss = log_loss(y_valid, pred_val)\nval_acc = accuracy_score(y_valid, np.round(pred_val))\nprint(f'valid loss: {val_loss}\\t valid accuracy: {val_acc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b7d7134eb16a4dc822587ee3488f918638eab23","collapsed":true},"cell_type":"code","source":"pred_test = model.predict_generator(test_flow, steps=test_steps, verbose=1)\npred_test = pred_test.ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"960893bdef553227d9bd8b7b26a753d33d0d1575","collapsed":true},"cell_type":"code","source":"np.save('valid_pred.npy', pred_val)\nnp.save('test_pred.npy', pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afa954364b87ccf86b16ecee05f62ce3ed97ced4","collapsed":true},"cell_type":"code","source":"evals.loc[evals['is_test']==1, 'img_id'].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad1dfbfd1b90687c265de84e602224e633878a5a","collapsed":true},"cell_type":"code","source":"mask = evals['is_test']==1\nsub = {\n    'id': evals.loc[mask, 'img_id'].values.astype('int'),\n    'label': pred_test,\n}\nsub = pd.DataFrame(sub).sort_values(by='id').reset_index(drop=True)\nsub['label'] = 1 - sub['label']\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0accc6756777fce86cb3cdb70f5dd002d93bbb21","collapsed":true},"cell_type":"code","source":"subname = f'basic_{val_loss:.6f}.csv'\nsub.to_csv(subname, index=False)\nprint(subname, 'saved')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"214c919ea31b7393e70d41c1185bc1bc762f5861"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}