{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Dog Breeding Classifications"},{"metadata":{},"cell_type":"markdown","source":"![](https://thehappypuppysite.com/wp-content/uploads/2017/07/lab.jpg)"},{"metadata":{"_uuid":"11aa55f3db0dec51c09334c64fae7f56b1a2cde1"},"cell_type":"markdown","source":"# 1. Import \n"},{"metadata":{"_uuid":"ae198668fcf74479c92b2dfbf2dc3f291b599d0e","trusted":true},"cell_type":"code","source":"# System\nimport sys\nimport os\nimport argparse\n\n# Time\nimport time\nimport datetime\n\n# Numerical Data\nimport random\nimport numpy as np \nimport pandas as pd\n\n# Tools\nimport shutil\nfrom glob import glob\nfrom tqdm import tqdm\nimport gc\n\n# NLP\nimport re\n\n# Preprocessing\nfrom sklearn import preprocessing\nfrom sklearn.utils import class_weight as cw\nfrom sklearn.utils import shuffle\n\n# Model Selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import cross_val_score\n\n# Machine Learning Models\nfrom sklearn import svm\nfrom sklearn.svm import LinearSVC, SVC\n\n# Evaluation Metrics\nfrom sklearn import metrics \nfrom sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score, confusion_matrix, classification_report, roc_auc_score\n\n\n# Deep Learning - Keras -  Preprocessing\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# Deep Learning - Keras - Model\nimport keras\nfrom keras import models\nfrom keras.models import Model\nfrom keras.models import Sequential\n\n# Deep Learning - Keras - Layers\nfrom keras.layers import Convolution1D, concatenate, SpatialDropout1D, GlobalMaxPool1D, GlobalAvgPool1D, Embedding, \\\n    Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten\nfrom keras.layers import Dense, Input, Dropout, MaxPool2D, MaxPooling2D, Concatenate, GlobalMaxPooling2D, GlobalAveragePooling2D, \\\n    Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D\nfrom keras.layers.pooling import _GlobalPooling1D\n\nfrom keras.regularizers import l2\n\n# Deep Learning - Keras - Pretrained Models\nfrom keras.applications.xception import Xception\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.applications.inception_resnet_v2 import InceptionResNetV2\nfrom keras.applications.densenet import DenseNet201\nfrom keras.applications.nasnet import NASNetMobile, NASNetLarge\n\nfrom keras.applications.nasnet import preprocess_input\n\n# Deep Learning - Keras - Model Parameters and Evaluation Metrics\nfrom keras import optimizers\nfrom keras.optimizers import Adam, SGD , RMSprop\nfrom keras.losses import mae, sparse_categorical_crossentropy, binary_crossentropy\n\n# Deep Learning - Keras - Visualisation\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau, LearningRateScheduler\n# from keras.wrappers.scikit_learn import KerasClassifier\nfrom keras import backend as K\n\n# Deep Learning - TensorFlow\nimport tensorflow as tf\n\n# Graph/ Visualization\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport matplotlib.image as mpimg\nimport seaborn as sns\nfrom mlxtend.plotting import plot_confusion_matrix\n\n# Image\nimport cv2\nfrom PIL import Image\nfrom IPython.display import display\n\n# np.random.seed(42)\n\n%matplotlib inline\n\n# Input data\nprint(os.listdir(\"../input/\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ec897e74117506f17611b58af485a2e013e848"},"cell_type":"markdown","source":"# 2. Functions"},{"metadata":{"_uuid":"cbe4bc8e76957ae5b139e87a19e75e51be7af414","trusted":true},"cell_type":"code","source":"def date_time(x):\n    if x==1:\n        return 'Timestamp: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())\n    if x==2:    \n        return 'Timestamp: {:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())\n    if x==3:  \n        return 'Date now: %s' % datetime.datetime.now()\n    if x==4:  \n        return 'Date today: %s' % datetime.date.today()  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc1f975dabb307ccf1c219b5021dfc4a97a7a117"},"cell_type":"markdown","source":"# 3. Input Configuration"},{"metadata":{"_uuid":"a8a36cda21a37ddf8002c994b682d91a81c59b56","trusted":true},"cell_type":"code","source":"input_directory = r\"../input/dog-breed-identification/\"\noutput_directory = r\"../output/\"\n\ntraining_dir = input_directory + \"train\"\ntesting_dir = input_directory + \"test\"\n\nif not os.path.exists(output_directory):\n    os.mkdir(output_directory)\n    \nfigure_directory = \"../output/figures\"\nif not os.path.exists(figure_directory):\n    os.mkdir(figure_directory)    \n    \nfile_name_pred_batch = figure_directory+r\"/result\"\nfile_name_pred_sample = figure_directory+r\"/sample\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bd626b286efbc7ca36291ad5b856c25d8c2da52","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(input_directory + \"/labels.csv\")\ntrain_df.rename(columns={\"breed\": \"label\"}, inplace=True)\ntrain_df[\"id\"] = train_df[\"id\"].apply(lambda x: x+\".\"+\"jpg\")\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"classes = list(train_df[\"label\"].unique())\nclasses.sort()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_files = os.listdir(testing_dir)\n    \ntest_df = pd.DataFrame({\"id\": test_files, \"label\": \"boston_bull\"})\n\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_df), len(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e02a5a1b4538b67bf351ec59a9cfd7f832fb0e3"},"cell_type":"markdown","source":"# 4. Visualization"},{"metadata":{"_uuid":"90fed2e4b122755b68312621acd1cbfa6bdd3ddd","trusted":true},"cell_type":"code","source":"def plot_image(file, directory=None, sub=False, aspect=None, title=False):\n    path = directory + \"/\" + file\n    \n    img = plt.imread(path)\n    \n    plt.imshow(img, aspect=aspect)\n    if title:\n        plt.title(file)\n    plt.xticks([])\n    plt.yticks([])\n    \n    if sub:\n        plt.show()\n        \ndef plot_img_dir(directory=training_dir, count=5):\n    selected_files = random.sample(os.listdir(directory), count)\n    \n    ncols = 5\n    nrows = count//ncols if count%ncols==0 else count//ncols+1\n    \n    figsize=(20, ncols*nrows)\n\n    ticksize = 14\n    titlesize = ticksize + 8\n    labelsize = ticksize + 5\n\n\n    params = {'figure.figsize' : figsize,\n              'axes.labelsize' : labelsize,\n              'axes.titlesize' : titlesize,\n              'xtick.labelsize': ticksize,\n              'ytick.labelsize': ticksize}\n\n    plt.rcParams.update(params)\n    \n    i=0\n    \n    for file in selected_files:        \n        plt.subplot(nrows, ncols, i+1)\n        path = directory + file\n        plot_image(file, directory, aspect=None)\n\n        i=i+1\n    \n    plt.tight_layout()\n    plt.show()\n    \ndef plot_img_df(directory=None, df=None, filename=\"id\", label = \"label\", count=5, num_cat=-1):\n    label_map = {}\n    \n    classes = list(set(df[label]))\n#     classes.sort()\n    \n    for l in classes:\n        label_map[l] = df[df[label]==l][filename]\n        label_map[l] = label_map[l].sample(count, replace=True)\n        \n    \n    \n    ncols = 5\n    nrows = count//ncols if count%ncols==0 else count//ncols+1\n#     print(nrows, ncols)\n    \n    figsize=(20, ncols*nrows)\n\n    ticksize = 14\n    titlesize = ticksize + 8\n    labelsize = ticksize + 5\n\n\n    params = {'figure.figsize' : figsize,\n              'axes.labelsize' : labelsize,\n              'axes.titlesize' : titlesize,\n              'xtick.labelsize': ticksize,\n              'ytick.labelsize': ticksize}\n\n    plt.rcParams.update(params)\n    \n    i=0\n    if num_cat==-1:\n        print(\"Showing {} classes...\".format(len(label_map)))\n    else:\n        print(\"Showing {} classes...\".format(num_cat))\n        \n    for label in label_map:\n        if num_cat==i:\n            break\n        label2 = re.sub(\"_\", \" \", label)\n        label2 = label2.title()\n        print(str(i+1) + \". \" + label2)\n        \n        j=0\n        for id, file in label_map[label].iteritems():\n            plt.subplot(nrows, 5, j+1)\n            plot_image(file, directory, aspect='auto')\n            j=j+1\n            \n        plt.tight_layout()\n        plt.show()\n        \n        \n        i+=1\n        \ndef plot_img_dir_main(directory=training_dir, count=5):\n    labels = os.listdir(directory)\n    \n    for label in labels:\n        label2 = re.sub(\"_\", \" \", label)\n        label2 = label2.title()\n        print(label2)\n        plot_img_dir(directory=directory+\"/\"+label, count=count)\n        num_cat-=1\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.1 Show image samples of all classes/ selected number of classes"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_img_df(directory=training_dir, df=train_df, filename=\"id\", label = \"label\", count=5, num_cat=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 4.2 Countplot for Images of Categories"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df2 = train_df.copy()\ntrain_df2[\"label\"] = train_df[\"label\"].apply(lambda x: re.sub(\"_\", \" \", x))\ntrain_df2[\"label\"] = train_df2[\"label\"].apply(lambda x: x.title())\n\nclasses2 = train_df2[\"label\"].unique()\nclasses2.sort()\n\nrows = (len(classes) - 1)/4","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.2.1 Countplot Sorted by Value Count of Categories"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18, rows))\nax = sns.countplot(y=\"label\", data=train_df2, order=train_df2[\"label\"].value_counts().index)\nplt.title(\"Countplot sorted by Value count of Categories\")\nplt.tight_layout()\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.2.2 Countplot Sorted by Alphebetical Order of Category Names"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18, rows))\nax = sns.countplot(y=\"label\", data=train_df2, order=classes2)\nplt.title(\"Countplot Sorted by Alphebetical Order of Category Names\")\nplt.tight_layout()\nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08472c5647d039fe88134dd5bbf51b1bacafe700"},"cell_type":"markdown","source":"# 5. Preprocessing Function"},{"metadata":{"_uuid":"6368be0cd0b5b944d842b963bdfb9a8b22843510"},"cell_type":"markdown","source":"During Preprocessing, all of the image has been transformed to target size (299, 299) and pixel value has been rescaled to unit value. (299, 299) is the input shape for Pretrained model \"InceptionV3\". The target class is treated as categorical and both training and validation image set has been re-shuffled. Some of the images has been horizontally and vertically flipped randomly and sheerness and rotation has been changed to introduce heterogeneity. A part of training dataset has been used as validation set. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data(batch_size=32, target_size=(299, 299), class_mode=\"categorical\", training_dir=training_dir, testing_dir=testing_dir, x_col=\"id\", y_col=\"label\"):\n    print(\"Preprocessing and Generating Data Batches.......\\n\")\n    \n    rescale = 1.0/255\n\n    train_batch_size = batch_size\n    validation_batch_size = batch_size*5\n    test_batch_size = batch_size*5\n    \n    train_shuffle = True\n    val_shuffle = True\n    test_shuffle = False\n    \n    train_datagen = ImageDataGenerator(\n        horizontal_flip=True,\n#         vertical_flip=True,\n        rotation_range=45,\n        shear_range=15,\n        rescale=rescale,\n        validation_split=0.25)\n\n    train_generator = train_datagen.flow_from_dataframe(\n        train_df, \n        training_dir,\n        x_col=x_col,\n        y_col=y_col,  \n        target_size=target_size, \n#         classes=classes,\n        class_mode=class_mode, \n        batch_size=batch_size, \n        shuffle=True, \n        seed=42,\n        subset='training')\n    \n    \n    validation_generator = train_datagen.flow_from_dataframe(\n        train_df, \n        training_dir,\n        x_col=x_col,\n        y_col=y_col,  \n#         classes=classes,\n        target_size=target_size, \n        class_mode=class_mode, \n        batch_size=validation_batch_size, \n        shuffle=True, \n        seed=42,\n        subset='validation')\n    \n    test_datagen = ImageDataGenerator(rescale=rescale)\n    \n    test_generator = test_datagen.flow_from_dataframe(\n        test_df, \n        testing_dir,\n        x_col=x_col,\n        y_col=y_col,  \n#         classes=classes,\n        target_size=target_size, \n        class_mode=class_mode, \n        batch_size=test_batch_size, \n        shuffle=False, \n        seed=42)\n    \n#     y = [val for sublist in train_generator.classes for val in sublist]\n    class_weights = get_weight(train_generator.classes, binary=False)\n    \n    steps_per_epoch = len(train_generator)\n    validation_steps = len(validation_generator)\n    \n    print(\"\\nPreprocessing and Data Batch Generation Completed.\\n\")\n    \n    \n    return train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps\n\n# Calculate Class Weights\ndef get_weight(y, binary=True, n_samples=-1):\n    if binary==False:\n        class_weights =  cw.compute_class_weight('balanced', np.unique(y), y)\n    else:\n        d = {x:y.count(x) for x in set(y)}\n        num_class_temp = 2\n        class_weights = {}\n\n        for cls in d:\n            count = d.get(cls)\n            class_weight = n_samples / (num_class_temp * count)\n            class_weights[cls] = class_weight\n#             temp = np.concatenate((np.repeat(0, n_samples-count), np.repeat(1, count)))\n#             class_weight =  cw.compute_class_weight('balanced', np.unique(temp), temp)\n#             class_weights[cls] = class_weight\n    return class_weights\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e02af0cac288607e1ae124f8242db9b1c05688c8"},"cell_type":"markdown","source":"# 6. Model Function"},{"metadata":{},"cell_type":"markdown","source":"## 6.1 Pretrained Model"},{"metadata":{"_uuid":"868b5798867b18a80f8d5b12957cd7965a0b65ae","trusted":true},"cell_type":"code","source":"def get_model(model_name, input_shape=(96, 96, 3), num_class=2, weights='imagenet', dense_units=1024):\n    inputs = Input(input_shape)\n    \n    if model_name == \"Xception\":\n        base_model = Xception(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet50\":\n        base_model = ResNet50(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet101\":\n        base_model = keras.applications.resnet.ResNet101(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet152\":\n        base_model = keras.applications.resnet.ResNet152(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet50V2\":\n        base_model = resnet_v2.ResNet50V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet101V2\":\n        base_model = resnet_v2.ResNet101V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet152V2\":\n        base_model = resnet_v2.ResNet152V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNeXt50\":\n        base_model = resnext.ResNeXt50(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNeXt101\":\n        base_model = resnext.ResNeXt101(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"InceptionV3\":\n        base_model = InceptionV3(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"InceptionResNetV2\":\n        base_model = InceptionResNetV2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"DenseNet201\":\n        base_model = DenseNet201(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"NASNetMobile\":\n        base_model = NASNetMobile(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"NASNetLarge\":\n        base_model = NASNetLarge(include_top=False, weights=weights, input_shape=input_shape)\n        \n\n    x = base_model(inputs)\n    x = Dropout(0.8)(x) \n    x = GlobalAveragePooling2D()(x)\n#     x = Dropout(0.5)(x) \n    x = BatchNormalization()(x)\n    x = Dropout(0.8)(x)     \n\n    \n    if num_class>1:\n        outputs = Dense(num_class, activation=\"softmax\")(x)\n    else:\n        outputs = Dense(1, activation=\"sigmoid\")(x)\n\n#     model = Model(inputs=base_model.input, outputs=outputs)\n    model = Model(inputs=inputs, outputs=outputs)\n\n    model.summary()\n    \n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 6.2 Model Build from Scratch"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_conv_model(num_class=2, input_shape=(150,150, 3)):\n    model = Sequential()\n\n    model.add(Conv2D(32, (3, 3), input_shape=(32, 32, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(MaxPool2D())\n    model.add(Dropout(0.2))\n\n    model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Activation('relu'))\n    model.add(MaxPool2D())\n    model.add(Dropout(0.2))\n\n    model.add(GlobalAveragePooling2D())\n    \n    if num_class>1:\n        model.add(Dense(num_class, activation='softmax'))\n    else:\n        model.add(Dense(num_class, activation='sigmoid'))\n    \n#     print(model.summary())\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e56fcfe7a60a2759f1952f30f7fb05726b022c5"},"cell_type":"markdown","source":"# 7. Output Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"main_model_dir = output_directory + r\"models/\"\nmain_log_dir = output_directory + r\"logs/\"\n\ntry:\n    os.mkdir(main_model_dir)\nexcept:\n    print(\"Could not create main model directory\")\n    \ntry:\n    os.mkdir(main_log_dir)\nexcept:\n    print(\"Could not create main log directory\")\n\n\n\nmodel_dir = main_model_dir + time.strftime('%Y-%m-%d %H-%M-%S') + \"/\"\nlog_dir = main_log_dir + time.strftime('%Y-%m-%d %H-%M-%S')\n\n\ntry:\n    os.mkdir(model_dir)\nexcept:\n    print(\"Could not create model directory\")\n    \ntry:\n    os.mkdir(log_dir)\nexcept:\n    print(\"Could not create log directory\")\n    \nmodel_file = model_dir + \"{epoch:02d}-val_acc-{val_acc:.2f}-val_loss-{val_loss:.2f}.hdf5\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5133db17f6ff324823b131eacc3fb471943adc1"},"cell_type":"markdown","source":"# 8. Call Back Configuration"},{"metadata":{"_uuid":"104999913bc3bd3f1181b365fff5cc23f9a56ce7","trusted":true},"cell_type":"code","source":"print(\"Settting Callbacks\")\n\ndef step_decay(epoch, lr):\n    # initial_lrate = 1.0 # no longer needed\n    lrate = lr\n    if epoch==2:\n        lrate = 0.0001  \n#     lrate = lr * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n    return lrate\n\n\ncheckpoint = ModelCheckpoint(\n    model_file, \n    monitor='val_acc', \n    save_best_only=True)\n\n\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=3,\n    verbose=1,\n    restore_best_weights=True)\n\n\nreduce_lr = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=1,\n    min_lr=0.0000001,\n    verbose=1)\n\n\nlearning_rate_scheduler = LearningRateScheduler(step_decay, verbose=1)\n# f1_metrics = Metrics()\n\n\n\ncallbacks = [reduce_lr, early_stopping]\n# callbacks = [checkpoint, reduce_lr, early_stopping]\n# callbacks = [reduce_lr, early_stopping, f1_metrics]\n\n\nprint(\"Set Callbacks at \", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6fbe80b2f63f0b9972c1546b088b667331108aa5"},"cell_type":"markdown","source":"# 9. Load Model"},{"metadata":{},"cell_type":"markdown","source":"## 9.1 Model Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"# dim = 96\n# dim = 224\ndim = 299\n\ninput_shape = (dim, dim, 3)\n\nnum_class = len(classes)\n\nweights = 'imagenet'\ndense_units = 256","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 9.2 Load Model with Configuration"},{"metadata":{"_uuid":"27dc13b90ff17042f2912f50ef4a8ad288f9914f","trusted":true},"cell_type":"code","source":"print(\"Getting Base Model\", date_time(1))\n\nmodel = get_model(model_name=\"InceptionV3\", input_shape=input_shape, num_class=num_class, weights=weights, dense_units=dense_units)\n# model = get_conv_model(num_class=num_class, input_shape=input_shape)\nprint(\"Loaded Base Model\", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 10. Get Data"},{"metadata":{},"cell_type":"markdown","source":"## 10.1 Data Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64\n\nclass_mode = \"categorical\"\n# class_mode = \"binary\"\n\ntarget_size = (dim, dim)\ny_col = \"label\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c975187a17a5e9d07ff15484cac2824fcab1fd53"},"cell_type":"markdown","source":"## 10.2 Get Data"},{"metadata":{"_uuid":"6daf75886fbc7d444222398f04637e5cc4aed9e7","trusted":true},"cell_type":"code","source":"train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(batch_size=batch_size,\n                                                                                                                   target_size=target_size,\n                                                                                                                   class_mode=class_mode,\n                                                                                                                   y_col=y_col)    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 11. Model Compilation"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Compliling Model ...\")\n\n\nlearning_rate = 0.0001\noptimizer = Adam(learning_rate)\n# optimizer = Adam()\n\n\nloss = 'categorical_crossentropy'\n# loss = 'binary_crossentropy'\nmetrics = ['accuracy']\n# metrics = [auroc]\n\n\nmodel.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\nprint(\"Completed Model Compilation.\\n\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd11c2eb7719f3f866c9e9ee201ca67e556fc73b"},"cell_type":"markdown","source":"# 12. Training Model"},{"metadata":{"_uuid":"8e335384316089698ee7e1a2f2cd1d6ad2ed4df9"},"cell_type":"markdown","source":"Trained model on full tranning dataset for 50 epochs and validated on full validation dataset. Adjusted class weight has been used for trainning. For optimization used Adam optimizer with learning rate of 0.0001. For loss calculation used \"Categorical Crossentropy\" and for model performance evaluation used \"Accuracy\" as metrics.  "},{"metadata":{},"cell_type":"markdown","source":"## 12.1 Training Configuration"},{"metadata":{"trusted":true},"cell_type":"code","source":"steps_per_epoch = len(train_generator)\nvalidation_steps = len(validation_generator)\n\nverbose = 1\nepochs = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# batch_size = 32\n# train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Starting Trainning ...\\n\")\n\nstart_time = time.time()\nprint(date_time(1))\n\n\nprint(\"Trainning Model ...\\n\")\n\n\nhistory = model.fit_generator(\n    train_generator,\n    steps_per_epoch=steps_per_epoch,\n    epochs=epochs,\n    verbose=verbose,\n    callbacks=callbacks,\n    validation_data=validation_generator,\n    validation_steps=validation_steps, \n    class_weight=class_weights)\n\n\n\nelapsed_time = time.time() - start_time\nelapsed_time = time.strftime(\"%H:%M:%S\", time.gmtime(elapsed_time))\n\n\nprint(\"\\nElapsed Time: \" + elapsed_time)\nprint(\"Completed Model Trainning\", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36b7d58a68aa0603ac98b4f3fdcad627b918642d"},"cell_type":"markdown","source":"# 13. Model Performance Visualization\nModel Performance  Visualization over the Epochs"},{"metadata":{},"cell_type":"markdown","source":"## 13.1 Model Performance Visualization Function"},{"metadata":{"_uuid":"361892313801cd84bc66a62e5347c20c6c86e6fe","trusted":true},"cell_type":"code","source":"def plot_performance(history=None, figure_directory=None):\n    xlabel = 'Epoch'\n    legends = ['Training', 'Validation']\n\n#     ylim_pad = [0.005, 0.005]\n    ylim_pad = [0, 0]\n\n\n    plt.figure(figsize=(20, 5))\n\n    # Plot training & validation Accuracy values\n\n    y1 = history.history['acc']\n    y2 = history.history['val_acc']\n\n    min_y = min(min(y1), min(y2))-ylim_pad[0]\n    max_y = max(max(y1), max(y2))+ylim_pad[0]\n    \n#     min_y = .96\n#     max_y = 1\n\n\n    plt.subplot(121)\n\n    plt.plot(y1)\n    plt.plot(y2)\n\n    plt.title('Model Accuracy\\n'+date_time(1), fontsize=17)\n    plt.xlabel(xlabel, fontsize=15)\n    plt.ylabel('Accuracy', fontsize=15)\n    plt.ylim(min_y, max_y)\n    plt.legend(legends, loc='upper left')\n    plt.grid()\n\n\n    # Plot training & validation loss values\n\n    y1 = history.history['loss']\n    y2 = history.history['val_loss']\n\n    min_y = min(min(y1), min(y2))-ylim_pad[1]\n    max_y = max(max(y1), max(y2))+ylim_pad[1]\n\n#     min_y = .1\n#     max_y = 0\n\n    plt.subplot(122)\n\n    plt.plot(y1)\n    plt.plot(y2)\n\n    plt.title('Model Loss\\n'+date_time(1), fontsize=17)\n    plt.xlabel(xlabel, fontsize=15)\n    plt.ylabel('Loss', fontsize=15)\n    plt.ylim(min_y, max_y)\n    plt.legend(legends, loc='upper left')\n    plt.grid()\n    if figure_directory:\n        plt.savefig(figure_directory+\"/history\")\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 13.2 Visualization"},{"metadata":{"_uuid":"84f9360eb9554faaeaae16fbd5e9994b9c550c76","trusted":true},"cell_type":"code","source":"plot_performance(history=history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 14. Model Evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.evaluate_generator(generator=validation_generator, steps=len(validation_generator), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 15. Prediction and Inference"},{"metadata":{"trusted":true},"cell_type":"code","source":"label_map = (train_generator.class_indices)\nlabel_map_inv = {v: k for k, v in label_map.items()}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c09d411cfaed347995ec8308be13f2d848efe8","trusted":true},"cell_type":"code","source":"ypreds = model.predict_generator(generator=test_generator, steps = len(test_generator),  verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ypreds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f50d50f3e253212ed1083713ff6dcb16e893001","trusted":true},"cell_type":"code","source":"ypred = ypreds.argmax(axis=-1)\n# ypred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 16. Inference Performance Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_rand_test_img(test_generator=None, labels_test=None, count=5):\n\n    filepaths = test_generator.filepaths\n    file_names = test_generator.filenames\n\n    selected_filepaths = []\n    selected_file_names = []\n    selected_labels = []\n\n    mem = set()\n\n    i = count\n\n    num_test_sample = test_generator.n\n\n    for file in test_generator.labels:\n        if i<=0:\n            break\n\n        rnd = random.randint(0, test_generator.n)\n        while rnd in mem:\n            rnd = random.randint(0, test_generator.n)\n\n        selected_filepaths.append(filepaths[rnd])\n        selected_file_names.append(file_names[rnd])\n        \n        lbl = label_map_inv[labels_test[rnd]]\n        lbl = re.sub(\"_\", \" \", lbl)\n        lbl = lbl.title()\n\n        selected_labels.append(lbl)\n        \n        i-=1\n    \n    return selected_file_names, selected_file_names, selected_labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_img_count = 10\nlabels_test = ypred\nselected_file_names, selected_file_names, selected_labels = get_rand_test_img(test_generator=test_generator, labels_test=labels_test, count=test_img_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count = test_img_count\n\nncols = 5\nnrows = count//ncols if count%ncols==0 else count//ncols+1\n\nfigsize=(20, ncols*nrows)\n\nticksize = 14\ntitlesize = ticksize + 8\nlabelsize = ticksize + 5\n\n\nparams = {'figure.figsize' : figsize,\n          'axes.labelsize' : labelsize,\n          'axes.titlesize' : titlesize,\n          'xtick.labelsize': ticksize,\n          'ytick.labelsize': ticksize}\n\nplt.rcParams.update(params)\n\n\ni = 0\n\nfor i in range(0, count):\n    if i>test_img_count:\n        break    \n    plt.subplot(nrows, 5, i+1)\n    plot_image(selected_file_names[i], testing_dir, aspect='auto')\n    plt.title(selected_labels[i])\n\nplt.tight_layout()\nplt.show()     ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# count = 5\n\n# ncols = 5\n# nrows = count//ncols if count%ncols==0 else count//ncols+1\n\n# figsize=(20, ncols*nrows)\n\n# ticksize = 14\n# titlesize = ticksize + 8\n# labelsize = ticksize + 5\n\n\n# params = {'figure.figsize' : figsize,\n#           'axes.labelsize' : labelsize,\n#           'axes.titlesize' : titlesize,\n#           'xtick.labelsize': ticksize,\n#           'ytick.labelsize': ticksize}\n\n# plt.rcParams.update(params)\n\n\n# for label in set(test_generator.classes):\n#     print(label_map_inv[label])\n#     x = np.where(ypred == 1)[0].tolist()\n#     s = np.random.choice(x, count)\n#     j=0\n#     for index in s:\n#         file = test_generator.filenames[index]\n#         print(file)\n#         plt.subplot(nrows, 5, j+1)\n#         plot_image(file, testing_dir, aspect='auto')\n#         j=j+1\n\n#     plt.tight_layout()\n#     plt.show()\n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 17. Submission "},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(input_directory+\"sample_submission.csv\")\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir_files = os.listdir(testing_dir)\ntest_gen_files = test_generator.filenames\nsample_submission_files = sample_submission[\"id\"]\nlen(test_dir_files), len(test_gen_files), len(sample_submission_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m = {}\nl = len(test_gen_files)\nfor i in range(l):\n    m[test_gen_files[i]] = ypreds[i] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = (train_generator.class_indices)\nlabels = list(labels.keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ypreds_sync = []\nfor f in sample_submission_files:\n    ypreds_sync.append(m[f+\".jpg\"]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.DataFrame(data=ypreds_sync, columns=labels)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"id\"]  = sample_submission_files\n\ncols = test_df.columns.tolist()\ncols = cols[-1:] + cols[:-1]\ntest_df = test_df[cols]\n\ntest_df.to_csv('submission.csv', index=False)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}