{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Start by adding data\nAdd competition data from dog breed identification.","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check TensorFlow version","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:32:57.322753Z","iopub.execute_input":"2021-10-13T13:32:57.323007Z","iopub.status.idle":"2021-10-13T13:33:01.627374Z","shell.execute_reply.started":"2021-10-13T13:32:57.32294Z","shell.execute_reply":"2021-10-13T13:33:01.624617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os # read and manipulate local files\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport cv2\n\nfrom tensorflow.keras.applications import xception\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import Input\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import Callback, EarlyStopping\nfrom tensorflow.keras.layers import GlobalAveragePooling2D, Dense, Dropout, Conv2D, MaxPooling2D, Dense, Flatten, BatchNormalization, Concatenate","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:36:46.931496Z","iopub.execute_input":"2021-10-13T13:36:46.932306Z","iopub.status.idle":"2021-10-13T13:36:47.060531Z","shell.execute_reply.started":"2021-10-13T13:36:46.932263Z","shell.execute_reply":"2021-10-13T13:36:47.059829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Identify and read files","metadata":{}},{"cell_type":"code","source":"train_dir = '../input/dog-breed-identification/train/'\ntest_dir = '../input/dog-breed-identification/test/'\ntrain_labels = pd.read_csv('../input/dog-breed-identification/labels.csv')\nsubmission = pd.read_csv('../input/dog-breed-identification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:20.732946Z","iopub.execute_input":"2021-10-13T13:33:20.733266Z","iopub.status.idle":"2021-10-13T13:33:21.288211Z","shell.execute_reply.started":"2021-10-13T13:33:20.73323Z","shell.execute_reply":"2021-10-13T13:33:21.287364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_size = len(os.listdir(train_dir))\ntest_size = len(os.listdir(test_dir))\n\nprint(train_size,test_size)\nprint(train_labels.shape)\nprint(submission.shape)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:24.092799Z","iopub.execute_input":"2021-10-13T13:33:24.093117Z","iopub.status.idle":"2021-10-13T13:33:24.669354Z","shell.execute_reply.started":"2021-10-13T13:33:24.093085Z","shell.execute_reply":"2021-10-13T13:33:24.668519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare DataFrame for image book keeping","metadata":{}},{"cell_type":"code","source":"target, dog_breeds = pd.factorize(train_labels['breed'], sort = True) # Encode labels as numbers\ntrain_labels['target'] = target\ndisplay(train_labels.head()) # Image id, breed and coded target\ndisplay(submission.head())   # Submission format for Kaggle","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:26.845965Z","iopub.execute_input":"2021-10-13T13:33:26.846591Z","iopub.status.idle":"2021-10-13T13:33:26.901456Z","shell.execute_reply.started":"2021-10-13T13:33:26.846557Z","shell.execute_reply":"2021-10-13T13:33:26.900716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of representatives\ntrain_labels['breed'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:27.028737Z","iopub.execute_input":"2021-10-13T13:33:27.028939Z","iopub.status.idle":"2021-10-13T13:33:27.039416Z","shell.execute_reply.started":"2021-10-13T13:33:27.028916Z","shell.execute_reply":"2021-10-13T13:33:27.038613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DataFrame df combining image location with id and '.jpg' for direct access, concatenated with target labels\nlabels = []\ndata = []\nfor i in range(train_labels.shape[0]):\n    data.append(train_dir + train_labels['id'].iloc[i]+'.jpg')\n    labels.append(train_labels['target'].iloc[i])\ndf = pd.DataFrame(data)\ndf.columns = ['images']\ndf['target'] = labels\nprint(df.shape)\ndisplay(df.head())\n\ndel labels\ndel data","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:31.567483Z","iopub.execute_input":"2021-10-13T13:33:31.568062Z","iopub.status.idle":"2021-10-13T13:33:31.813295Z","shell.execute_reply.started":"2021-10-13T13:33:31.568024Z","shell.execute_reply":"2021-10-13T13:33:31.812501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Same for test data\ntest_data = []\nfor i in range(submission.shape[0]):\n    test_data.append(test_dir + submission['id'].iloc[i]+'.jpg')\ndf_test = pd.DataFrame(test_data)\ndf_test.columns = ['images']\nprint(df_test.shape)\ndisplay(df_test.head())\n\ndel test_data","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:33:32.009161Z","iopub.execute_input":"2021-10-13T13:33:32.009385Z","iopub.status.idle":"2021-10-13T13:33:32.137115Z","shell.execute_reply.started":"2021-10-13T13:33:32.009363Z","shell.execute_reply":"2021-10-13T13:33:32.136378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize some doggos","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize = (15,15))\nfor i in range(9):\n    plt.subplot(330 + 1 + i) # Shorthand for size 3x3, position i\n    image = cv2.imread(df['images'].iloc[i])\n    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # Reverse RGB order\n    plt.imshow(image)\n    plt.title(train_labels['breed'].iloc[i])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-13T13:40:22.427085Z","iopub.execute_input":"2021-10-13T13:40:22.427471Z","iopub.status.idle":"2021-10-13T13:40:24.257755Z","shell.execute_reply.started":"2021-10-13T13:40:22.427339Z","shell.execute_reply":"2021-10-13T13:40:24.257094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split data into train and test (validation)","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(df['images'],df['target'], stratify = df['target'], test_size=0.2, random_state=1234)\n\ntrain = pd.DataFrame(X_train)\ntrain.columns = ['images']\ntrain['target'] = y_train\n\nprint(train.shape)\ndisplay(train.head())\n\nvalidation = pd.DataFrame(X_val)\nvalidation.columns = ['images']\nvalidation['target'] = y_val\n\nprint(validation.shape)\ndisplay(validation.head())\n\ndel X_train, X_val, y_train, y_val","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:29:59.785606Z","iopub.execute_input":"2021-10-13T11:29:59.786216Z","iopub.status.idle":"2021-10-13T11:29:59.830911Z","shell.execute_reply.started":"2021-10-13T11:29:59.786186Z","shell.execute_reply":"2021-10-13T11:29:59.829657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set hypervariables","metadata":{}},{"cell_type":"code","source":"N_EPOCHS = 50\nBATCH_SIZE = 32\nIMG_HEIGHT = 299\nIMG_WIDTH = 299","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:30:00.860695Z","iopub.execute_input":"2021-10-13T11:30:00.861374Z","iopub.status.idle":"2021-10-13T11:30:00.868426Z","shell.execute_reply.started":"2021-10-13T11:30:00.861329Z","shell.execute_reply":"2021-10-13T11:30:00.865104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Augmentation and batch generation","metadata":{}},{"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    horizontal_flip=True, \n    zoom_range=0.2)\n\nval_datagen = ImageDataGenerator()\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    train,\n    x_col='images',\n    y_col='target',\n    target_size=(IMG_HEIGHT, IMG_WIDTH),\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    class_mode='raw')\n\nvalidation_generator = val_datagen.flow_from_dataframe(\n    validation,\n    x_col='images',\n    y_col='target',\n    target_size=(IMG_HEIGHT, IMG_WIDTH),\n    shuffle=False,\n    batch_size=BATCH_SIZE,\n    class_mode='raw')","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:30:02.455371Z","iopub.execute_input":"2021-10-13T11:30:02.455663Z","iopub.status.idle":"2021-10-13T11:30:32.607649Z","shell.execute_reply.started":"2021-10-13T11:30:02.455634Z","shell.execute_reply":"2021-10-13T11:30:32.606625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Basic modelling","metadata":{}},{"cell_type":"code","source":"# Define an inception cell\ndef inception_cell(x):\n    # 1x1 convolution\n    x1 = Conv2D(filters=32, kernel_size=(1,1), strides=(1,1), padding=\"same\", activation=\"relu\")(x)\n\n    # 1x1 + 3x3 convolution\n    x3 = Conv2D(filters=32, kernel_size=(1,1), strides=(1,1), padding=\"same\", activation=\"relu\")(x)\n    x3 = Conv2D(filters=32, kernel_size=(3,3), strides=(1,1), padding=\"same\", activation=\"relu\")(x3)\n\n    # 1x1 + 5x5 convolution\n    x5 = Conv2D(filters=32, kernel_size=(1,1), strides=(1,1), padding=\"same\", activation=\"relu\")(x)\n    x5 = Conv2D(filters=32, kernel_size=(5,5), strides=(1,1), padding=\"same\", activation=\"relu\")(x5)\n    \n    # MaxPool + 1x1 convolution\n    xp = MaxPooling2D(pool_size=(3,3), strides=(1,1), padding=\"same\")(x)\n    xp = Conv2D(filters=32, kernel_size=(1,1), strides=(1,1), padding=\"same\", activation=\"relu\")(xp)\n \n    x = Concatenate()([x1, x3, x5, xp])\n    return(x)\n","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:30:32.609613Z","iopub.execute_input":"2021-10-13T11:30:32.610071Z","iopub.status.idle":"2021-10-13T11:30:32.621435Z","shell.execute_reply.started":"2021-10-13T11:30:32.610022Z","shell.execute_reply":"2021-10-13T11:30:32.620246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = Input(shape=(IMG_HEIGHT, IMG_WIDTH, 3))\nx = Conv2D(filters=8, kernel_size=(3,3), padding=\"same\", activation=\"relu\")(inputs)\nx = MaxPooling2D(pool_size=(3,3))(x)\nx = inception_cell(x)\nx = BatchNormalization()(x)\nx = MaxPooling2D(pool_size=(3,3))(x)\nx = inception_cell(x)\nx = BatchNormalization()(x)\nx = MaxPooling2D(pool_size=(3,3))(x)\nx = Flatten()(x)\nx = Dropout(.5)(x)\noutputs = Dense(120, activation='softmax')(x)\nmodel = Model(inputs, outputs)\n\ndisplay(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:30:41.412653Z","iopub.execute_input":"2021-10-13T11:30:41.412955Z","iopub.status.idle":"2021-10-13T11:30:44.15513Z","shell.execute_reply.started":"2021-10-13T11:30:41.412918Z","shell.execute_reply":"2021-10-13T11:30:44.154172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transfer learning","metadata":{}},{"cell_type":"code","source":"# Xception model\nbase_model = xception.Xception(weights='imagenet', include_top=False, input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)) \n\n# Frozen\nbase_model.trainable = False\n\n# Convert input data to format excepted by Xception model\n# https://www.tensorflow.org/api_docs/python/tf/keras/applications/xception/preprocess_input\ninputs = Input(shape=(IMG_HEIGHT, IMG_WIDTH, 3))\nx = xception.preprocess_input(inputs) \n\n# The base model contains batchnorm layers. We want to keep them in inference mode\n# when we unfreeze the base model for fine-tuning, so we make sure that the\n# base_model is running in inference mode here by passing `training=False`.\nx = base_model(x, training=False)\n\nx = GlobalAveragePooling2D()(x)\n\n#     adding extra dense layer\n#     x = Dense(1024, activation='relu')(x)\n#     x = Dropout(.7)(x)\n#     x = Dense(512, activation='relu')(x)\n\nx = Dropout(.5)(x)\noutputs = Dense(120, activation='softmax')(x)\nmodel = Model(inputs, outputs)\n\ndisplay(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-10-13T09:57:56.126743Z","iopub.execute_input":"2021-10-13T09:57:56.127185Z","iopub.status.idle":"2021-10-13T09:58:00.056636Z","shell.execute_reply.started":"2021-10-13T09:57:56.127149Z","shell.execute_reply":"2021-10-13T09:58:00.055954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adam optimiser and sparse categorical crossentropy","metadata":{}},{"cell_type":"code","source":"optimizer = Adam(learning_rate=0.001)\nmodel.compile(loss=\"sparse_categorical_crossentropy\", metrics=['accuracy'], optimizer=optimizer)\nn_train_steps = train.shape[0]//BATCH_SIZE\nn_val_steps = validation.shape[0]//BATCH_SIZE\nprint(\"Number of training and validation steps: {} and {}\".format(n_train_steps, n_val_steps))","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:31:55.362975Z","iopub.execute_input":"2021-10-13T11:31:55.363464Z","iopub.status.idle":"2021-10-13T11:31:55.386071Z","shell.execute_reply.started":"2021-10-13T11:31:55.363433Z","shell.execute_reply":"2021-10-13T11:31:55.384557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Early stopping\nwhen small change (delta)","metadata":{}},{"cell_type":"code","source":"EarlyStop_callback = EarlyStopping(min_delta=0.001, patience=10, restore_best_weights=True)\n\n# # DECREASE LEARNING RATE EACH EPOCH\n# annealer = LearningRateScheduler(lambda epoch: 1e-5 * 0.95 ** epoch, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:31:58.30118Z","iopub.execute_input":"2021-10-13T11:31:58.303913Z","iopub.status.idle":"2021-10-13T11:31:58.308594Z","shell.execute_reply.started":"2021-10-13T11:31:58.303882Z","shell.execute_reply":"2021-10-13T11:31:58.307369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Go!","metadata":{}},{"cell_type":"code","source":"model.fit(\n    train_generator,\n    epochs=N_EPOCHS,\n    validation_data=validation_generator,\n    callbacks=[EarlyStop_callback],\n    )","metadata":{"execution":{"iopub.status.busy":"2021-10-13T11:32:01.834522Z","iopub.execute_input":"2021-10-13T11:32:01.83485Z","iopub.status.idle":"2021-10-13T12:46:51.295688Z","shell.execute_reply.started":"2021-10-13T11:32:01.834806Z","shell.execute_reply":"2021-10-13T12:46:51.294732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_generator, validation_generator","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test set predictions\n(if used for Kaggle competition)","metadata":{}},{"cell_type":"code","source":"test_datagen = ImageDataGenerator()\n\ntest_generator = test_datagen.flow_from_dataframe(\n    df_test,\n    x_col='images',\n    target_size=(IMG_HEIGHT, IMG_WIDTH),\n    shuffle=False,\n    batch_size=BATCH_SIZE,\n    class_mode=None\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict(\n    test_generator,\n    verbose=1         \n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(predictions.shape)\nprint(predictions)","metadata":{},"execution_count":null,"outputs":[]}]}