{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport glob\nimport matplotlib.pyplot as plt\nimport shutil\nfrom tqdm import tqdm\nimport cv2\nimport time\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport gc\nimport random\nimport re\n#os.mkdir('../data/')\nprint(os.listdir(\"..\"))\n# Any results you write to the current directory are saved as output.\n\nfrom keras import backend\nfrom keras.applications.inception_v3 import InceptionV3,preprocess_input\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import SGD\nfrom keras.models import Model, load_model\nfrom keras.layers import Dense, GlobalAveragePooling2D\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce0371a3dea52dff677a6606524709378a6f73e8"},"cell_type":"markdown","source":"#### **We' ll try to train our model using an already existing Conv NN called InceptionV3 model using Transfer Learning. We'll internally move the files to arrange them in subfolders in order to make use of flow_from_directory() function.** \n### **This will help use to train the model in batch mode instead of loading every thing in the memory.**"},{"metadata":{"_uuid":"3ccb13b101c85187b2e1841aacc5ea106f229e7c"},"cell_type":"markdown","source":"## Preprocessing and Loading of data"},{"metadata":{"trusted":true,"_uuid":"9888c859c317d0ce22f88296f4aae6e20ad507bf"},"cell_type":"code","source":"#Total labels to classify\nlabels=['dog','cat']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90eadfd1e546271f7eef94e35153c8c911501128"},"cell_type":"code","source":"#Helper functions to resuhffle data\n#Keeping all new data in ../data directory as ../input ir RO.\ndef create_directory(labels):\n    src_dir='../data'\n    if not os.path.exists(src_dir+'/train'):\n        print('Creating Directory', src_dir+'/train')\n        os.makedirs(src_dir+'/train',exist_ok=True)\n    else:\n        print(src_dir+'/train', ' exists!')\n    \n        #functions to create directory with the label names\n    for l in labels:\n        dir_path=r''+src_dir+'/train/'+l\n        if not os.path.exists(dir_path):\n            print('Creating Directory',dir_path)\n            os.mkdir(dir_path)\n        else:\n            print(dir_path, ' exists!')\n                \n    if not os.path.exists(src_dir+'/test/test_data'):\n        print('Creating Directory', src_dir+'/test/test_data')\n        os.makedirs(src_dir+'/test/test_data',exist_ok=True)\n    else:\n        print(src_dir+'/test/test_data', ' exists!')\n        \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe2b7a6ebba559010d9731710b71d8bc0a7cce87"},"cell_type":"code","source":"create_directory(labels =labels)\n#shutil.rmtree(path='../data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f6902b88136e63425885bcb2ae60db45984c319"},"cell_type":"code","source":"#Function to move the images tp their corresponding folders:\ndef move_files(train_path,test_path):\n    print('Moving Training Files ..')\n    time.sleep(1)\n    for i in tqdm(os.listdir(train_path)):        \n        if 'dog' in i:\n            shutil.copyfile(train_path+i,'../data/train/dog/'+i )\n        elif 'cat' in i:\n            shutil.copyfile(train_path+i,'../data/train/cat/'+i )\n        else:\n            print('unkown File', i)\n            \n    print('Moving Testing Files ..')\n    time.sleep(1)\n    for i in tqdm(os.listdir(test_path)):                \n        shutil.copyfile(test_path+i,'../data/test/test_data/'+i )\n        \n    #print('File Copy in complete!')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22f915120d5cccbdcd4e77646b0e89868236d60d"},"cell_type":"code","source":"move_files('../input/train/','../input/test/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0768275b15d6fe41b1bc0334321309f1bca8ebcb"},"cell_type":"code","source":"# Get count of number of files in this folder and all subfolders\ndef get_num_files(path):\n    if not os.path.exists(path):\n        return 0\n    return sum([len(files) for r, d, files in os.walk(path)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b75db5a6d4ffc6acbdef3b831d1827a718d0648"},"cell_type":"code","source":"#Setting Image and model parameters\nImage_width,Image_height = 299,299\nbatch_size=50\nNumber_FC_Neurons=1024\ntrain_dir = '../data/train/'\n\ntotal_samples = get_num_files(train_dir)\nval_split=0.3\nn_train=total_samples*(1-val_split)\nn_val=total_samples*val_split\nnum_classes = len(labels)\nprint(n_train,n_val)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Define data pre-processing \ntrain_image_gen = ImageDataGenerator(rescale=1/255,horizontal_flip=True,validation_split=val_split)\n'''train_image_gen = ImageDataGenerator(rescale=1/255,\n        preprocessing_function=preprocess_input,\n        horizontal_flip=True,\n        validation_split=0.3\n    )'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d11fd3a1997f8e3ccd85052e7336a14db9995bb6"},"cell_type":"code","source":"train_generator = train_image_gen.flow_from_directory(train_dir,target_size=(Image_width,Image_height),batch_size=batch_size,seed=42,subset='training',shuffle=True,class_mode='categorical')\nval_generator = train_image_gen.flow_from_directory(train_dir,target_size=(Image_width,Image_height),batch_size=batch_size,seed=42,subset='validation',shuffle=True,class_mode='categorical')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1d9e851928fd84be1e3831a2d74f1382c96fa3c"},"cell_type":"code","source":"#Load the inception model and load with its pre trained weight. But exclude the last layer aa we would train that.\n\nInceptionV3_base_model = InceptionV3(weights='imagenet', include_top=False)    #To exclude final conv layer \nprint('Inception v3 base model without last FC loaded')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f2282ff6625646c20b026ca46eb8fcae546f7073"},"cell_type":"code","source":"#Defining the new Final conv layers \n#Using Functional APIs\nx = InceptionV3_base_model.output\nx_pool = GlobalAveragePooling2D()(x)\nx_dense = Dense(Number_FC_Neurons,activation='relu')(x_pool)\nfinal_pred = Dense(num_classes,activation='softmax')(x_dense)\nmodel = Model(inputs=InceptionV3_base_model.input,outputs=final_pred)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a2776d629a000533404c089ec9cf2e30480ce70"},"cell_type":"code","source":"#Adding Callback For early stopping.\n#We use validation loss as the monitoring parameter which we need to minimize.\n#Also, we'll use the result from the best epoch, thus, restore_best_weights=True(Default is False)\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint,TensorBoard\n\ncb_checkpoint = ModelCheckpoint(filepath = '../working/best.hd5', monitor = 'val_loss', save_best_only = True, mode = 'auto',)\ncb_stopping = EarlyStopping(monitor='val_loss',patience=10,mode=min,restore_best_weights=True)\ncb_tensorboard = TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True,write_images=False)\nmy_callback=[cb_stopping, cb_checkpoint,cb_tensorboard]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5762b77cc77754d13e802529081dfbd3c989fad8"},"cell_type":"code","source":"#Freeze all the layers in InceptionV3 model to train only our additional layers\n'''for layer in InceptionV3_base_model.layers:\n    layer.trainable=False\n'''\n#Fine tune model by retraining the few end layers of the inception model\nlayer_to_Freeze=172    \nfor layer in model.layers[:layer_to_Freeze]:\n    layer.trainable =False\nfor layer in model.layers[layer_to_Freeze:]:\n    layer.trainable=True\n#Define model compile for basic transfer learning\n#Using categorical_crossentropy loss as we need to classify only 2 classes and using softmax output layer.\n#Please read difference between categorical_crossentropy and binary_crossentropy here:= https://stackoverflow.com/questions/47877083/keras-binary-crossentropy-categorical-crossentropy-confusion\nsgd = SGD(lr = 0.01, decay = 1e-6, momentum = 0.9, nesterov = True)\nmodel.compile(optimizer=sgd,loss='binary_crossentropy',metrics=['accuracy'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"534e19f1a6141cc250b7f76253af17045f5ceb4a"},"cell_type":"code","source":"# Fit the transfer learning model to the data from the generators.  \n# Since the data is coming from the generators that we created ,we use fit_generator() method. \n# And we need to specify the train and val_generator as the source of data.\n# Please note the difference between fit() and fit_generator()\n\nhistory_transfer_learning = model.fit_generator(train_generator,epochs=30,\n                                                steps_per_epoch=n_train//batch_size,\n                                                validation_data=val_generator,\n                                                validation_steps=n_val//batch_size,\n                                                verbose=1,\n                                                callbacks=my_callback,\n                                                class_weight='auto')\n\n#Always save your trained model, as training process is tedious. It's better to train once and then load the already trained model.\n#model.save('model.hd5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3f8ce29b5cc9f377977aadc4d887fb67f634718"},"cell_type":"code","source":"#incase you have already trained your model, then you can load it by using following Function.\n#Loading the best model.\nfrom keras.models import load_model\nprint('Loading best model...')\nbest_model = load_model('../working/best.hd5')\nprint('Best model loaded!')\n#os.listdir('../working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c7d9102aebcbc29c350dbf8d95b23e6433615da"},"cell_type":"code","source":"#Clearing memory leak\n#backend.clear_session()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99bc7291499f6b2c62f549c9f862f54bfafe1f9a"},"cell_type":"code","source":"#Evaluating Metrics on Validation set\n#y_pred = model.predict_generator(val_generator,verbose=1)\n\n#Converting probability to target\n#y_pred[y_pred > 0.5]=1\n#y_pred[y_pred < 0.5]=0\n\n#Use Evaluate() or evalute_generator() to check your model accuracy on validation set.\nscore = best_model.evaluate_generator(val_generator,verbose=1)\nprint('Test loss: ', score[0])\nprint('Test accuracy', score[1])\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c95cf114a39db624dcf0c1111c43e8ea8cba499d"},"cell_type":"code","source":"epoch_list = list(range(1,len(history_transfer_learning.history['acc'])+1))  #Values for x axis[1,2,3,4...# of epochs]\nplt.plot(epoch_list, history_transfer_learning.history['acc'],epoch_list,history_transfer_learning.history['val_acc'])\nplt.legend(('Training accuracy','Validation Accuracy'))\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55f066b480fbdc8a76df92fbbec68277904eb52b"},"cell_type":"code","source":"\nepoch_list = list(range(1,len(history_transfer_learning.history['loss'])+1))  #Values for x axis[1,2,3,4...# of epochs]\nplt.plot(epoch_list, history_transfer_learning.history['loss'],epoch_list,history_transfer_learning.history['val_loss'])\nplt.legend(('Training loss','Validation loss'))\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fff1f76b20ab26383b5d31d5ad5957affbacdc43"},"cell_type":"code","source":"test_dir='../data/test/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa3f8b7f138e25af457f720e893f379086b49d8e"},"cell_type":"code","source":"os.listdir('../working/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5aa1666c70c1530728db5f72ce6f11116fb98e3c"},"cell_type":"code","source":"# Define data pre-processing \ntest_image_gen = ImageDataGenerator(rescale=1/255)\ntest_generator = test_image_gen.flow_from_directory(test_dir,target_size=(Image_width,Image_height),batch_size=1,seed=42,class_mode=None,shuffle=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ae73b745ebba77e2d471ea5b5e3f9f8814ad354"},"cell_type":"code","source":"#test_generator.reset()\ny_pred = model.predict_generator(generator=test_generator,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed2db00d45d6c2f0b467aadbfdf65e19f92a69ff"},"cell_type":"code","source":"#submission = pd.DataFrame({'id':np.arange(1,len(test_generator.filenames)+1),'label':y_pred.clip(min=0.02,max=0.98)[:,1]})\nsubmission = pd.DataFrame({'id':pd.Series(test_generator.filenames),'label':pd.Series(y_pred.clip(min=0.02,max=0.98)[:,1])})\n#submission = pd.DataFrame({'id':pd.Series(test_generator.filenames),'label':pd.Series(y_pred[:,1])})\nsubmission['id'] = submission.id.str.extract('(\\d+)')\nsubmission['id']=pd.to_numeric(submission['id'])\n#submission.sort_values(by='id',inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3d1332ba484126c92d25bf950258323ca8e2f76"},"cell_type":"code","source":"#y_pred.clip(min=0.02,max=0.98)[:100,1]\n#score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"184d386df4de6932ed15b3fdc58df7e09f4f4ff6"},"cell_type":"code","source":"#submission.nunique(axis=0)\nsubmission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f97dedd63a04ed8843f4d017f703ed0a58a26fc2"},"cell_type":"code","source":"#submission.shape\n#y_pred[:15,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e491cb162394b629f25cc933a7801c289b5fb40c"},"cell_type":"code","source":"submission.to_csv('DogVsCats_submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94da6db4b929011b621fb3ef0518aaad747b2bb1"},"cell_type":"code","source":"from keras.preprocessing.image import load_img\n\nfig,ax = plt.subplots(5,5, figsize=(15,15))\nfor i,fn in enumerate(test_generator.filenames[:25]):\n    path='../data/test/'+fn\n    #print(path)    \n    #print(i)\n    img=load_img(path, target_size=(Image_width, Image_height))\n    ax[i//5, i%5].imshow(img)\n    ax[i//5, i%5].axis('off')\n    \n    if (submission['label'][i]) > 0.5:\n        label='Dog'\n    elif (submission['label'][i])<0.5:\n            label='Cat'\n    ax[i//5, i%5].set_title(\"It's a \"+label)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7045d58b13b592bffe43f58d55c4ce8956a446b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b11d191df4d105cbe34018fb23e73fb691146764"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cfa5afbfc73f9a194fd7152d857876c6b8c4bba"},"cell_type":"code","source":"len(model.layers)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}