{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob as glob\nfrom tensorflow.keras.callbacks import TensorBoard\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.layers import Conv2D, Input, Dense, MaxPooling2D, Activation,Flatten, Dropout, BatchNormalization, MaxPool2D\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam,RMSprop\nfrom keras.callbacks import EarlyStopping\nfrom tqdm.notebook import tqdm\nimport gc\nimport joblib\n\nimport os\nos.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2023-01-17T15:51:54.882129Z","iopub.execute_input":"2023-01-17T15:51:54.882919Z","iopub.status.idle":"2023-01-17T15:52:01.418644Z","shell.execute_reply.started":"2023-01-17T15:51:54.882816Z","shell.execute_reply":"2023-01-17T15:52:01.417514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reading the labels file to get the classes","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv('/kaggle/input/dog-breed-identification/labels.csv')\nlabels['new_id'] = labels['id'] + '.jpg'\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:01.420976Z","iopub.execute_input":"2023-01-17T15:52:01.421865Z","iopub.status.idle":"2023-01-17T15:52:01.492312Z","shell.execute_reply.started":"2023-01-17T15:52:01.421825Z","shell.execute_reply":"2023-01-17T15:52:01.491256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = list(labels.sort_values(by='breed')['breed'].unique())\nclasses[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:01.496909Z","iopub.execute_input":"2023-01-17T15:52:01.499325Z","iopub.status.idle":"2023-01-17T15:52:01.527986Z","shell.execute_reply.started":"2023-01-17T15:52:01.499287Z","shell.execute_reply":"2023-01-17T15:52:01.527023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"print('Total No. of breeds in the dataset :',labels.breed.unique().shape[0])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:01.533870Z","iopub.execute_input":"2023-01-17T15:52:01.536525Z","iopub.status.idle":"2023-01-17T15:52:01.555512Z","shell.execute_reply.started":"2023-01-17T15:52:01.536483Z","shell.execute_reply":"2023-01-17T15:52:01.554362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.breed.value_counts()[:120].plot(kind='bar',figsize=(30,5),title = 'Bar plot of Dogs breed and number of images',color='black')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:01.566080Z","iopub.execute_input":"2023-01-17T15:52:01.568242Z","iopub.status.idle":"2023-01-17T15:52:03.899077Z","shell.execute_reply.started":"2023-01-17T15:52:01.568207Z","shell.execute_reply":"2023-01-17T15:52:03.898133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"value_counts_name = labels.breed.value_counts().index\nvalue_counts = labels.breed.value_counts()\nprint(\"The breed with the highest no. of photos is :\",value_counts_name[0],\n      \"\\nNo.of Images : \",value_counts[0],\n      \"\\nThe breed with the lowest no. of photos is :\",value_counts_name[-1],\n      \"\\nNo.of Images :\",value_counts[-1])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:03.900234Z","iopub.execute_input":"2023-01-17T15:52:03.901396Z","iopub.status.idle":"2023-01-17T15:52:03.912525Z","shell.execute_reply.started":"2023-01-17T15:52:03.901355Z","shell.execute_reply":"2023-01-17T15:52:03.911252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Images per breed : {labels.shape[0]/labels.breed.unique().shape[0]}\")","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:03.914286Z","iopub.execute_input":"2023-01-17T15:52:03.914911Z","iopub.status.idle":"2023-01-17T15:52:03.922536Z","shell.execute_reply.started":"2023-01-17T15:52:03.914874Z","shell.execute_reply":"2023-01-17T15:52:03.921432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.isnull().sum().sum()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:03.924130Z","iopub.execute_input":"2023-01-17T15:52:03.924927Z","iopub.status.idle":"2023-01-17T15:52:03.936161Z","shell.execute_reply.started":"2023-01-17T15:52:03.924882Z","shell.execute_reply":"2023-01-17T15:52:03.935201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are 10k plus photos of Dogs that are classified in 120 breeds, this comes down to on average of 85 images of dogs per breed with which we should first try and fit a model, but before that we need to split the training data in train and split and check how the model is performing. \n\nWe can take actions as following:\n\n* Data Augmentation\n* Transfer Learning","metadata":{}},{"cell_type":"markdown","source":"# Image Generator","metadata":{}},{"cell_type":"code","source":"def image_generator(ImageShape):\n    batch = 32\n    datagen = ImageDataGenerator(rescale= 1./255, validation_split=0.2, rotation_range=20,\n                                    zoom_range=0.1, width_shift_range=0.2, height_shift_range=0.2,\n                                    shear_range=0.1, horizontal_flip=True, fill_mode=\"nearest\")\n    \n    train = datagen.flow_from_dataframe(dataframe = labels,\n                                        x_col = 'new_id',\n                                        y_col = 'breed',\n                                        target_size = (ImageShape,ImageShape),\n                                        directory = '/kaggle/input/dog-breed-identification/train/',\n                                        batch_size = batch,\n                                        subset = 'training',\n                                        class_mode = 'categorical',\n                                        shuffle = True, \n                                        seed = 12\n    )\n\n    validation = datagen.flow_from_dataframe(dataframe = labels,\n                                             x_col = 'new_id',\n                                             y_col = 'breed',\n                                             target_size = (ImageShape,ImageShape),\n                                             directory='/kaggle/input/dog-breed-identification/train/',\n                                             batch_size = batch ,\n                                             shuffle=True,\n                                             class_mode='categorical', \n                                             seed = 12, \n                                             subset = 'validation'\n    )\n    \n    return train,validation\n\n","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:03.939258Z","iopub.execute_input":"2023-01-17T15:52:03.939608Z","iopub.status.idle":"2023-01-17T15:52:03.949097Z","shell.execute_reply.started":"2023-01-17T15:52:03.939582Z","shell.execute_reply":"2023-01-17T15:52:03.948367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"GPUS = [\"GPU:0\",\"GPU:1\"]\nstrategy = tf.distribute.MirroredStrategy( GPUS )\nprint('Number of devices: %d' % strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:03.954174Z","iopub.execute_input":"2023-01-17T15:52:03.954761Z","iopub.status.idle":"2023-01-17T15:52:08.115592Z","shell.execute_reply.started":"2023-01-17T15:52:03.954727Z","shell.execute_reply":"2023-01-17T15:52:08.114612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train,validation = image_generator(224)\n\n\nfor x in train:\n    data1 = x[0]\n    data2 = x[1]\n    print(data1.shape)\n    print()\n    print(data2.shape)\n    break\n\nplt.subplots(8,4, figsize = (20,20))\nfor i in range(1,33):\n    plt.subplot(8,4,i)\n    plt.imshow(data1[i-1])\n    plt.title(classes[np.argmax(data2[i-1])])\n    plt.tick_params(left = False, right = False , labelleft = False ,\n                labelbottom = False, bottom = False)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:08.116916Z","iopub.execute_input":"2023-01-17T15:52:08.117787Z","iopub.status.idle":"2023-01-17T15:52:37.837261Z","shell.execute_reply.started":"2023-01-17T15:52:08.117748Z","shell.execute_reply":"2023-01-17T15:52:37.835330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n\n    model.add(Input(shape = (224,224,3)))\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), activation ='relu'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(Conv2D(filters = 16, kernel_size = (3,3), activation ='relu'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(MaxPool2D(pool_size=(2,2)))\n\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), activation ='relu', kernel_regularizer = 'l2'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(Conv2D(filters = 16, kernel_size = (3,3), activation ='relu'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(MaxPool2D(pool_size=(2,2)))\n\n    model.add(Conv2D(filters = 32, kernel_size = (3,3), activation ='relu', kernel_regularizer = 'l2'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(Conv2D(filters = 8, kernel_size = (3,3), activation ='relu'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(MaxPool2D(pool_size=(2,2)))\n\n    model.add(Conv2D(filters = 16, kernel_size = (3,3), activation ='relu', kernel_regularizer = 'l2'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(Conv2D(filters = 8, kernel_size = (3,3), activation ='relu'))\n    model.add(tf.keras.layers.BatchNormalization())\n    model.add(MaxPool2D(pool_size=(2,2)))\n\n    model.add(Flatten())\n    model.add(Dense(256, activation = \"relu\", kernel_regularizer = 'l2'))\n    model.add(Dense(150, activation = \"relu\", kernel_regularizer = 'l2'))\n    model.add(Dense(120, activation = \"softmax\"))\n\n    model.compile(loss = 'categorical_crossentropy', optimizer = Adam(0.0001),metrics=['accuracy'])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:37.838469Z","iopub.execute_input":"2023-01-17T15:52:37.839559Z","iopub.status.idle":"2023-01-17T15:52:38.484332Z","shell.execute_reply.started":"2023-01-17T15:52:37.839516Z","shell.execute_reply":"2023-01-17T15:52:38.483421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename='CNN_log.csv'\nhistory_logger=tf.keras.callbacks.CSVLogger(filename, separator=\",\", append=True)\n\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=3)","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:38.485574Z","iopub.execute_input":"2023-01-17T15:52:38.486390Z","iopub.status.idle":"2023-01-17T15:52:38.492841Z","shell.execute_reply.started":"2023-01-17T15:52:38.486349Z","shell.execute_reply":"2023-01-17T15:52:38.491810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train, epochs = 20, validation_data = validation,callbacks = [history_logger, es])","metadata":{"execution":{"iopub.status.busy":"2023-01-17T15:52:38.494461Z","iopub.execute_input":"2023-01-17T15:52:38.495296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our Model is not able to learn and predict for the available data, So we can use prtrained model and use them for problem to identify different breeds of dog, We will try this with VGG16 , Inception and Xception model, first we will fit the model without training the pre learned weights once the model has converged to a little lower loss we will train the whole model even the prelearned weights with a very small learning rate which can imporve the performance of our model","metadata":{}},{"cell_type":"markdown","source":"# Transfer Learning","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.applications import VGG16","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    base_model = VGG16(include_top = False,weights ='imagenet',input_shape=(128,128,3))\n    base_model.trainable=False\n    vgg_model = Sequential()\n    vgg_model.add(Input(shape = (128,128,3)))\n    vgg_model.add(base_model)\n    vgg_model.add(Flatten())\n    vgg_model.add(Dense(256))\n    vgg_model.add(tf.keras.layers.Activation('relu'))\n    vgg_model.add(Dense(120,activation='softmax'))\n    \n    \n    vgg_model.compile(optimizer=tf.keras.optimizers.Adam(),metrics='accuracy',loss='categorical_crossentropy')\n\n    \nvgg_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename='vgg_log.csv'\nhistory_logger=tf.keras.callbacks.CSVLogger(filename, separator=\",\", append=True)\n\n\nvgg_train,vgg_val = image_generator(128)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x,y in vgg_train:\n    plt.imshow(x[0])\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model.fit(vgg_train, epochs = 25, validation_data=vgg_val, callbacks = [history_logger, es])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.trainable=True\nwith strategy.scope():\n        vgg_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate = 0.00001),metrics='accuracy',loss='categorical_crossentropy')\n\nvgg_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg_model.fit(vgg_train, epochs = 10, validation_data=vgg_val, callbacks = [history_logger, es])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### VGG Model performed better than the model that was created from scratch but it did start overfitting and was not able to generalize let's try thr Inception v3 model.","metadata":{}},{"cell_type":"markdown","source":"# Inception","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    base_model = tf.keras.applications.InceptionV3(weights=\"imagenet\",input_shape=(224,224,3),include_top = False)\n    base_model.trainable = False\n\n    Incep_model = Sequential()\n    Incep_model.add(Input(shape = (224,224,3)))\n    Incep_model.add(base_model)\n    Incep_model.add(tf.keras.layers.GlobalMaxPooling2D())\n    Incep_model.add(Dense(256, activation = 'relu', kernel_regularizer = 'l2'))\n    Incep_model.add(Dense(128, activation = 'relu', kernel_regularizer = 'l2'))\n    Incep_model.add(Dense(120, activation = 'softmax'))\n\n    Incep_model.compile(optimizer=tf.keras.optimizers.Adam(),metrics='accuracy',loss='categorical_crossentropy')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Incep_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"filename='incep_log.csv'\nhistory_logger=tf.keras.callbacks.CSVLogger(filename, separator=\",\", append=True)\n\nincep_train, incep_val = image_generator(224)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x, y in incep_train:\n    plt.imshow(x[0])\n    print(classes[np.argmax(y)])\n    break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Incep_model.fit(incep_train, epochs = 25, validation_data=incep_val, callbacks = [history_logger, es])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model.trainable = True\nIncep_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    Incep_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.00001),metrics='accuracy',loss='categorical_crossentropy')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Incep_model.fit(incep_train, epochs = 5, validation_data=incep_val, callbacks = [history_logger, es])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The inception model performed better than vgg model and we have achived a great accuracy, now we will try the same approach with the xception model.","metadata":{}},{"cell_type":"markdown","source":"# Xception","metadata":{}},{"cell_type":"code","source":"es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=3)\n\nfilename='xcep_log.csv'\nhistory_logger=tf.keras.callbacks.CSVLogger(filename, separator=\",\", append=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds,val_ds = image_generator(299)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(train_ds[0][0][0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    base_model = tf.keras.applications.Xception(include_top = False,weights = 'imagenet',input_shape = (299,299,3))\n    base_model.trainable = False\n    xcep_model = Sequential()\n    xcep_model.add(base_model)\n    xcep_model.add(tf.keras.layers.GlobalMaxPooling2D())\n    xcep_model.add(Dense(512,activation = 'relu'))\n    xcep_model.add(Dropout(0.5))\n    xcep_model.add(Dense(120,activation = 'softmax'))\n    \n    xcep_model.compile(loss=\"categorical_crossentropy\", metrics=['accuracy'], optimizer=tf.keras.optimizers.Adam(learning_rate = 0.0004))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xcep_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xcep_model.fit(train_ds, validation_data=val_ds, epochs=30, callbacks = [es, history_logger])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    base_model.trainable = True\n    xcep_model.compile(loss=\"categorical_crossentropy\", metrics=['accuracy'], optimizer=tf.keras.optimizers.Adam(learning_rate = 0.00001))\n    \nxcep_model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.compat.v1 import ConfigProto\nfrom tensorflow.compat.v1 import InteractiveSession\nconfig = ConfigProto()\nconfig.gpu_options.allow_growth = True\nsession = InteractiveSession(config=config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xcep_model.fit(train_ds, validation_data=val_ds, epochs=10, callbacks=[es, history_logger], batch_size = 64)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xcep_model.save('xcep_model.hdf5')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting the Outputs on test data","metadata":{}},{"cell_type":"code","source":"test_images_files_names = os.listdir('/kaggle/input/dog-breed-identification/test')\ntest_df = pd.DataFrame(test_images_files_names, columns=['id'])\ntest_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_datagen = ImageDataGenerator(rescale= 1./255)\ntest = test_datagen.flow_from_dataframe(test_df, directory='/kaggle/input/dog-breed-identification/test', x_col='id',target_size=(299,299), class_mode=None, batch_size=64, shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = xcep_model.predict(test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.DataFrame(columns=[\"id\"] + classes)\nresults","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results[\"id\"] = [os.path.splitext(file)[0] for file in os.listdir('/kaggle/input/dog-breed-identification/test')]\nresults.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results[classes] = preds\nresults.head()\n\nresults.to_csv(\"submission.csv\",index=False)\nresults","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}