{"cells":[{"metadata":{},"cell_type":"markdown","source":"### - First, i give all credits to every Kaggle Grandmasters like xhulu, Chris Deotte, CPMP etc (i can't list all of them one by one), also to everyone who shares their notebook on Kaggle.\n\n### - I just copy and learn from them how to make a Deep Learning model.\n\nThis is just for fun :)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport random\nimport math\nimport cv2\nimport os\nimport gc\n\nimport tensorflow as tf\nimport tensorflow.keras.backend as K\nprint('Use Tensorflow version:', tf.__version__)\n\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.layers import Dense, Lambda, Input\nfrom tensorflow.keras.models import Model\n\nplt.style.use('fivethirtyeight')\nplt.rcParams['figure.figsize'] = [16, 8]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def set_seed(seed: int):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    \nset_seed(33)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For tf.dataset\nAUTO = tf.data.experimental.AUTOTUNE\n\n# Configuration\nEPOCHS = 12\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_train_img(mode):\n    \n    fig, ax = plt.subplots(nrows=2, ncols=5, figsize=(24, 10))\n    \n    train_path = f'/kaggle/input/dog-breed-identification/{mode}/'\n    ten_random = pd.Series(os.listdir(train_path)).sample(10).values\n    \n    for idx, image in enumerate(ten_random):\n        final_path = os.path.join(train_path, image)\n        img = cv2.imread(final_path)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        \n        ax.ravel()[idx].imshow(img)\n        ax.ravel()[idx].axis('off')\n        \n    plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_train_img('train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_train_img('test')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/dog-breed-identification/labels.csv')\nsub = pd.read_csv('/kaggle/input/dog-breed-identification/sample_submission.csv')\n\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"unique_value = df['breed'].nunique()\n\nprint(f'We have {unique_value} dog categories in this dataset')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\n# don't forget to add .jpg\ntrain_paths = [os.path.join(GCS_DS_PATH, 'train', id + '.jpg') for id in df['id']]\ntrain_paths = np.array(train_paths)\n\ntest_paths = [os.path.join(GCS_DS_PATH, 'test', id + '.jpg') for id in sub['id']]\ntest_paths = np.array(test_paths)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dog_categories = np.sort(df['breed'].unique())\n\ncat2idx = {}\nidx2cat = {}\n\nfor idx, category in enumerate(dog_categories):\n    cat2idx[category] = idx\n    idx2cat[idx] = category","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\n\nlabel_encoded = df['breed'].map(cat2idx)\ntrain_labels = to_categorical(label_encoded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_paths, valid_paths, train_labels, valid_labels = train_test_split(train_paths,\n                                                                        train_labels,\n                                                                        test_size=0.12,\n                                                                        stratify=train_labels,\n                                                                        random_state=2020)\n\nprint(f'train size: {train_paths.shape[0]} images')\nprint(f'validation size {valid_paths.shape[0]} images')\nprint(f'\\ntest size {sub.shape[0]} images')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Image Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mat(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):\n    # returns 3x3 transformmatrix which transforms indicies\n        \n    # CONVERT DEGREES TO RADIANS\n    rotation = math.pi * rotation / 180.\n    shear    = math.pi * shear    / 180.\n\n    def get_3x3_mat(lst):\n        return tf.reshape(tf.concat([lst],axis=0), [3,3])\n    \n    # ROTATION MATRIX\n    c1   = tf.math.cos(rotation)\n    s1   = tf.math.sin(rotation)\n    one  = tf.constant([1],dtype='float32')\n    zero = tf.constant([0],dtype='float32')\n    \n    rotation_matrix = get_3x3_mat([c1,   s1,   zero, \n                                   -s1,  c1,   zero, \n                                   zero, zero, one])    \n    # SHEAR MATRIX\n    c2 = tf.math.cos(shear)\n    s2 = tf.math.sin(shear)    \n    \n    shear_matrix = get_3x3_mat([one,  s2,   zero, \n                                zero, c2,   zero, \n                                zero, zero, one])        \n    # ZOOM MATRIX\n    zoom_matrix = get_3x3_mat([one/height_zoom, zero,           zero, \n                               zero,            one/width_zoom, zero, \n                               zero,            zero,           one])    \n    # SHIFT MATRIX\n    shift_matrix = get_3x3_mat([one,  zero, height_shift, \n                                zero, one,  width_shift, \n                                zero, zero, one])\n    \n    return K.dot(K.dot(rotation_matrix, shear_matrix), \n                 K.dot(zoom_matrix,     shift_matrix))\n\n\ndef transform(image):    \n    # input image - is one image of size [dim,dim,3] not a batch of [b,dim,dim,3]\n    # output - image randomly rotated, sheared, zoomed, and shifted\n    \n    DIM = 480\n    XDIM = DIM%2 #fix for size 331\n    \n    rot = 180.0 * tf.random.normal([1], dtype='float32')\n    shr = 2.0 * tf.random.normal([1], dtype='float32') \n    h_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n    w_zoom = 1.0 + tf.random.normal([1], dtype='float32') / 8.0\n    h_shift = 8.0 * tf.random.normal([1], dtype='float32') \n    w_shift = 8.0 * tf.random.normal([1], dtype='float32') \n\n    # GET TRANSFORMATION MATRIX\n    m = get_mat(rot, shr, h_zoom, w_zoom, h_shift, w_shift) \n\n    # LIST DESTINATION PIXEL INDICES\n    x   = tf.repeat(tf.range(DIM//2, -DIM//2,-1), DIM)\n    y   = tf.tile(tf.range(-DIM//2, DIM//2), [DIM])\n    z   = tf.ones([DIM*DIM], dtype='int32')\n    idx = tf.stack( [x,y,z] )\n    \n    # ROTATE DESTINATION PIXELS ONTO ORIGIN PIXELS\n    idx2 = K.dot(m, tf.cast(idx, dtype='float32'))\n    idx2 = K.cast(idx2, dtype='int32')\n    idx2 = K.clip(idx2, -DIM//2+XDIM+1, DIM//2)\n    \n    # FIND ORIGIN PIXEL VALUES           \n    idx3 = tf.stack([DIM//2-idx2[0,], DIM//2-1+idx2[1,]])\n    d    = tf.gather_nd(image, tf.transpose(idx3))\n        \n    return tf.reshape(d,[DIM, DIM, 3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_image(filename, label=None, augment=True, final_size=456):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.image.resize(image, (480, 480)) # 1\n    image = tf.cast(image, tf.float32) / 255.0\n    \n    if augment:\n        \n        image = transform(image)\n        image = tf.image.random_crop(image, [468, 468, 3]) # 2\n        image = tf.image.random_flip_left_right(image)\n        image = tf.image.random_brightness(image, max_delta=0.2)\n        image = tf.image.random_contrast(image, lower=0.8, upper=1.2)\n        image = tf.image.random_saturation(image, lower=0.75, upper=1.2)\n    \n    image = tf.image.resize(image, (final_size, final_size))\n    image = tf.reshape(image, [final_size, final_size, 3]) # 3\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decode_image(filename, label=None, image_size=(456, 456)):\n    bits = tf.io.read_file(filename)\n    image = tf.image.decode_jpeg(bits, channels=3)\n    image = tf.cast(image, tf.float32) / 255.0\n    image = tf.image.resize(image, image_size)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def data_augment(image, label=None):\n    image = tf.image.random_flip_left_right(image)\n    image = tf.image.random_brightness(image, max_delta=0.2)\n    image = tf.image.random_contrast(image, lower=0.8, upper=1.2)\n    image = tf.image.random_saturation(image, lower=0.75, upper=1.2)\n    \n    if label is None:\n        return image\n    else:\n        return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((train_paths, train_labels))\n    .map(prepare_image, num_parallel_calls=AUTO)\n    .cache()\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((valid_paths, valid_labels))\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(test_paths)\n    .map(decode_image, num_parallel_calls=AUTO)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q efficientnet\n\nfrom efficientnet.tfkeras import EfficientNetB5\nimport tensorflow_addons as tfa","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras.backend as K\n\ndef categorical_focal_loss_with_label_smoothing(gamma=2.0, alpha=0.25, ls=0.1, classes=120.0):\n    \"\"\"\n    Implementation of Focal Loss from the paper in multiclass classification\n    Formula:\n        loss = -alpha*((1-p)^gamma)*log(p)\n        y_ls = (1 - α) * y_hot + α / classes\n    Parameters:\n        alpha -- the same as wighting factor in balanced cross entropy\n        gamma -- focusing parameter for modulating factor (1-p)\n        ls    -- label smoothing parameter(alpha)\n        classes     -- No. of classes\n    Default value:\n        gamma -- 2.0 as mentioned in the paper\n        alpha -- 0.25 as mentioned in the paper\n        ls    -- 0.1\n        classes     -- 42\n    \"\"\"\n    def focal_loss(y_true, y_pred):\n        # Define epsilon so that the backpropagation will not result in NaN\n        # for 0 divisor case\n        epsilon = K.epsilon()\n        # Add the epsilon to prediction value\n        #y_pred = y_pred + epsilon\n        #label smoothing\n        y_pred_ls = (1 - ls) * y_pred + ls / classes\n        # Clip the prediction value\n        y_pred_ls = K.clip(y_pred_ls, epsilon, 1.0-epsilon)\n        # Calculate cross entropy\n        cross_entropy = -y_true*K.log(y_pred_ls)\n        # Calculate weight that consists of  modulating factor and weighting factor\n        weight = alpha * y_true * K.pow((1-y_pred_ls), gamma)\n        # Calculate focal loss\n        loss = weight * cross_entropy\n        # Sum the losses in mini_batch\n        loss = K.sum(loss, axis=1)\n        return loss\n    \n    return focal_loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.Sequential([\n        EfficientNetB5(weights='imagenet', # noisy-student\n                       include_top=False,\n                       pooling='avg'),\n        Dense(120, activation='softmax')\n    ])\n    \n#     model.layers[0].trainable = False\n    \n    model.compile(optimizer = 'adam', # tfa.optimizers.AdamW(weight_decay=1e-4)\n                  loss = categorical_focal_loss_with_label_smoothing(gamma=2.0, alpha=0.75, ls=0.125, classes=120.0),\n                  metrics=['categorical_accuracy'])\n    \n    model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"LR = 0.0002 # 0.0005\nEPOCHS = 12\nWARMUP = 4\n\ndef get_cosine_schedule_with_warmup(lr, num_warmup_steps, num_training_steps, num_cycles=0.5):\n    \"\"\"\n    Modified the get_cosine_schedule_with_warmup from huggingface for tensorflow\n    (https://huggingface.co/transformers/_modules/transformers/optimization.html#get_cosine_schedule_with_warmup)\n\n    Create a schedule with a learning rate that decreases following the\n    values of the cosine function between 0 and `pi * cycles` after a warmup\n    period during which it increases linearly between 0 and 1.\n    \"\"\"\n\n    def lrfn(epoch):\n        if epoch < num_warmup_steps:\n            return float(epoch) / float(max(1, num_warmup_steps)) * lr\n        progress = float(epoch - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps))\n        return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) * lr\n\n    return tf.keras.callbacks.LearningRateScheduler(lrfn, verbose=True)\n\nlr_schedule= get_cosine_schedule_with_warmup(lr=LR, num_warmup_steps=WARMUP, num_training_steps=EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = train_labels.shape[0] // BATCH_SIZE  # 8995 / 128 = 70\n\nhistory = model.fit(\n    train_dataset, \n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS,\n    callbacks=[lr_schedule]\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get training and test loss histories\ntraining_loss = history.history['loss']\ntest_loss = history.history['val_loss']\n\n# Create count of the number of epochs\nepoch_count = range(1, len(training_loss) + 1)\n\n# Visualize loss history\nplt.plot(epoch_count, training_loss, 'r--')\nplt.plot(epoch_count, test_loss, 'b-')\nplt.legend(['Training Loss', 'Validation Loss'])\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset_tta = (\n        tf.data.Dataset\n        .from_tensor_slices(test_paths)\n        .map(decode_image, num_parallel_calls=AUTO)\n        .cache()\n        .map(data_augment, num_parallel_calls=AUTO)\n        .batch(BATCH_SIZE)\n)\n\ntta_times = 5\nprobabilities = []\n\nfor i in range(tta_times):\n    print('TTA Number: ', i + 1, '\\n')\n    probabilities.append(model.predict(test_dataset_tta, verbose=1))\n    \ntta_pred = np.mean(probabilities, axis=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"for cat in cat2idx.keys():\n    sub[cat] = tta_pred[:, cat2idx[cat]]\nsub.to_csv('submission.csv', index=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Got score 0.38639 for both public and private LB, not bad :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}