{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Import Library"},{"metadata":{},"cell_type":"markdown","source":"* number of training samples: 8000  (4000 cat - 4000 dog)\n* number of validation samples: 1600 (800 cat - 800 dog)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing.image import ImageDataGenerator, load_img\nfrom keras import optimizers\nfrom keras.models import Sequential\nfrom keras.layers import Dropout, Flatten, Dense,GlobalAveragePooling2D\nfrom keras import applications\nfrom pathlib import Path\nfrom keras.models import model_from_json\nfrom keras.callbacks import ModelCheckpoint, History\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nimport matplotlib.pyplot as plt\nimport random\nimport os","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"train\")\n\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\",\"r\") as zip_ref:\n    zip_ref.extractall(\"test1\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_directory = \"train/train/\"\ntest_directory  = \"test1/test1/\"\n# See sample image\nfilenames = os.listdir(train_directory)\nsample = random.choice(filenames)\nprint(sample)\nimage = load_img(train_directory + sample)\nplt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# 8000 train samples\n# 1600 validation samples\nimport shutil\nsource_dir = 'train/'\ndef copy_files(prefix_str, range_start, range_end, target_dir):\n    image_paths = []\n    for i in range(range_start, range_end):\n        image_path = os.path.join(source_dir,'train', prefix_str + '.'+ str(i)+ '.jpg')\n        image_paths.append(image_path)\n    dest_dir = os.path.join( 'data', target_dir, prefix_str)\n    os.makedirs(dest_dir)\n\n    for image_path in image_paths:\n        shutil.copy(image_path,  dest_dir)\n\ncopy_files('dog', 0, 4000, 'train')\ncopy_files('cat', 0, 4000, 'train')\ncopy_files('dog', 4000, 4800,'validation')\ncopy_files('cat', 4000, 4800, 'validation')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# All data, 12500 cat, 12500 dog\nsource_dir = 'train/'\ndef copy_files(prefix_str, range_start, range_end, target_dir):\n    image_paths = []\n    for i in range(range_start, range_end):\n        image_path = os.path.join(source_dir,'train', prefix_str + '.'+ str(i)+ '.jpg')\n        image_paths.append(image_path)\n    dest_dir = os.path.join( 'Alldata', target_dir, prefix_str)\n    if not os.path.exists(dest_dir):\n        os.makedirs(dest_dir)\n\n    for image_path in image_paths:\n        shutil.copy(image_path,  dest_dir)\n\ncopy_files('dog', 0, 12500, 'train')\ncopy_files('cat', 0, 12500, 'train')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#remove train folder\nif  os.path.exists('train'):\n    #os.removedirs(\"train\")\n    shutil.rmtree(\"train\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing Library"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Learning curves\ndef Polt_history(hist):\n    acc = hist.history['accuracy']\n    val_acc = hist.history['val_accuracy']\n\n    loss = hist.history['loss']\n    val_loss = hist.history['val_loss']\n    print(\"Accuracy = %0.3f\" % (acc[epochs-1]*100),  \", val_acc = %0.3f\" % (val_acc[epochs-1]*100))\n    print(\"loss     = %0.3f\" % loss[epochs-1], \", val_loss= %0.3f\" % val_loss[epochs-1])\n    plt.figure(figsize=(8, 8))\n    plt.subplot(2, 1, 1)\n    plt.plot(acc, label='Training Accuracy')\n    plt.plot(val_acc, label='Validation Accuracy')\n    plt.legend(loc='lower right')\n    plt.ylabel('Accuracy')\n    plt.ylim([min(plt.ylim()),1])\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(2, 1, 2)\n    plt.plot(loss, label='Training Loss')\n    plt.plot(val_loss, label='Validation Loss')\n    plt.legend(loc='upper right')\n    plt.ylabel('Cross Entropy')\n    plt.ylim([0,1.0])\n    plt.title('Training and Validation Loss')\n    plt.xlabel('epoch')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model predict\ndef Model_predict(Model,Test_dir):  \n    test_filenames = []\n    for file in os.listdir(Test_dir):   \n        test_filenames.append(os.path.join(Test_dir,file))  \n\n    test_df = pd.DataFrame({\n        'filename': test_filenames\n    })\n\n    test_datagen = ImageDataGenerator(rescale=1./255)\n    test_generator=test_datagen.flow_from_dataframe(\n                dataframe=test_df,\n                x_col=\"filename\",\n                y_col=None,\n                batch_size=50,\n                seed=42,\n                shuffle=False,\n                class_mode=None,\n                target_size=(img_height,img_width))\n    \n    nb_test_samples = len(test_df)\n    test_steps=nb_test_samples // 50\n    pred=Model.predict_generator(test_generator,\n                    steps=test_steps,\n                    verbose=1)\n    \n    pred = [1 if p[0] > 0.5 else 0 for p in pred]\n    print (pred[:12])\n    #predicted_class_indices=np.argmax(pred,axis=1)\n    predicted_class_indices=np.argmax(pred)\n\n    #len(predicted_class_indices)\n    #print(predicted_class_indices[:12])\n    return pred,test_df\n    #return predicted_class_indices,test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing known data in train folder: on 25000 image \ndef Test_Model_known_Data(Model):\n    print(\"Testing cats....\")\n    model_pred_cat,test_df  = Model_predict(Model,\"Alldata/train/cat\") #0\n    print(\"Testing dogs....\")\n    model_pred_dog,test_df  = Model_predict(Model,\"Alldata/train/dog\") #1\n\n    #print result\n    model_true_cat  = len(test_df) - sum (model_pred_cat)\n    model_true_dog  = sum (model_pred_dog)\n    model_true      = model_true_cat + model_true_dog\n    # model result\n    print(\"  model result\")\n    print(\"cat accuracy  = %2.3f\" % (model_true_cat /len(test_df) *100))\n    print(\"dog accuracy  = %2.3f\" % (model_true_dog /len(test_df) *100))\n    print(\"Total accuracy= %2.3f\" % (model_true /(2*len(test_df)) *100))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# Plot predict image output\n%matplotlib inline\n#import matplotlib.image as mpimg\nimport matplotlib.pyplot as plt\n\ndef Plot_predict(predicted_class_indices,Test_dir,test_df):\n    # Parameters for our graph; we'll output images in a 4x4 configuration\n    nrows = 12\n    ncols = 4\n    pic_index = 0 # Index for iterating over images\n    # Set up matplotlib fig, and size it to fit 4x4 pics\n    fig = plt.gcf()\n    fig.set_size_inches(ncols*4, nrows*4)\n\n    for i, img_path in enumerate(test_df.filename[:48]):\n        # Set up subplot; subplot indices start at 1\n        sp = plt.subplot(nrows, ncols, i + 1)\n        sp.axis('Off') # Don't show axes (or gridlines)\n\n        #img = mpimg.imread(img_path, target_size=(256, 256))Test_dir\n        img = load_img( img_path, target_size=(150,150))\n        plt.imshow(img) \n        result = predicted_class_indices[i]\n        if (result == 1 ):\n            name = 'Dog'\n        else :\n            name = 'Cat'\n        plt.title( name )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save Submission to csv file\ndef Save_Submission(predict,model,mod,test_df):\n    if not os.path.exists(mod):\n        os.makedirs(mod)\n        \n    test_df['category'] = predict\n    submission_df = test_df.copy()\n    #submission_df['id'] = submission_df['filename'].str.split('.').str[0]\n    submission_df['id'] = submission_df['filename'].str.split('.').str[0].str.split('/').str[1]\n    submission_df['label'] = submission_df['category']\n    submission_df.drop(['filename', 'category'], axis=1, inplace=True)\n    submission_df.index += 1 \n    submission_df.to_csv( mod + '/submission_AM_'+ mod +'.csv', index=True)\n\n    #plt.figure(figsize=(10,5))\n    submission_df['label'].value_counts().plot.bar()\n    plt.title(\"(Test data , \"+mod + \" )\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# dimensions of our images.\nimg_width, img_height = 150, 150      #299,299\nIMG_SHAPE = (img_width, img_height, 3)\n\ntrain_data_dir = 'data/train'\nvalidation_data_dir = 'data/validation'\n\nnb_train_samples = 8000\nnb_validation_samples = 1600\nepochs = 5\nbatch_size = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prepare data augmentation configuration\ntrain_datagen = ImageDataGenerator(\n                rescale=1./255,\n                shear_range=0.2,\n                zoom_range=0.2,\n                horizontal_flip=True)\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_generator = train_datagen.flow_from_directory(\n                train_data_dir,\n                target_size=(img_height, img_width),\n                batch_size=batch_size,\n                seed=42,\n                class_mode='binary')\n\nvalidation_generator = test_datagen.flow_from_directory(\n                validation_data_dir,\n                target_size=(img_height, img_width),\n                batch_size=batch_size,\n                seed=42,\n                class_mode='binary')#binary  categorical","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model 1"},{"metadata":{"trusted":true},"cell_type":"code","source":"# build the InceptionV3 network\nbase_model = applications.InceptionV3(input_shape=IMG_SHAPE,\n                                      weights='imagenet',\n                                      include_top=False) #, pooling='average'\nprint(\"base_model.layers\", len(base_model.layers)) #311\n\n#Freeze the convolutional base\n#for layer in base_model.layers[:100]:\n#    layer.trainable = False\nfor layer in base_model.layers:\n    layer.trainable = True\n\n# build a classifier model to put on top of the convolutional model\ntop_model = Sequential()\ntop_model.add(Flatten(input_shape=base_model.output_shape[1:]))\ntop_model.add(Dense(256, activation='relu'))\ntop_model.add(Dropout(0.5))\ntop_model.add(Dense(1, activation='sigmoid'))\n\nmodel = Sequential()\nmodel.add(base_model)\nmodel.add(top_model)\n\n#base_model.summary()\n#top_model.summary()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"if not os.path.exists('model'):\n    os.makedirs(\"model\")\n\ndef model_train(learningRate, epochs, itration):\n# compile the model with a RMSprop optimizer and a very slow learning rate.\n    model.compile(loss='binary_crossentropy',  #categorical_crossentropy\n                  #optimizer=optimizers.SGD(lr=learningRate, momentum=0.9),\n                  optimizer=optimizers.RMSprop(lr=learningRate),\n                  metrics=['accuracy'])\n    save = 'model/model.weights.best_InceptionV3_' + itration + '.hdf5'\n    checkpointer = ModelCheckpoint(filepath= save, \n                                   verbose=1, save_best_only=True)\n    # fit the model\n    hist = model.fit_generator(\n            train_generator,\n            samples_per_epoch=nb_train_samples,\n            epochs=epochs,\n            validation_data=validation_generator,\n            validation_steps=nb_validation_samples // batch_size,\n            callbacks=[checkpointer] )\n    return hist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_epochs = 5\nlearningRate = 1e-4\nhist = model_train(learningRate, initial_epochs, '1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save neural network structure and weights\nmodel_structure = model.to_json()\nf = Path(\"model/model_structure_InceptionV3.json\")\nf.write_text(model_structure)\nmodel.save_weights(\"model/model_weights_InceptionV3_1.h5\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model output: Learning curves"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"Polt_history(hist)\nplt.savefig('model/hist.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Continue train: fine tune model "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# compile the model with a SGD/momentum optimizer and a very slow learning rate.\nepochs = 5\nlearningRate=1e-5\nhist_2 = model_train(learningRate, epochs, '2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_epochs = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save neural network weights\nmodel.save_weights(\"model/model_weights_InceptionV3_2.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Learning curves\nPolt_history(hist_2)\nplt.savefig('model/hist_2.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Polt_history_fineTune(hist1,hist2):\n    acc      = hist1.history['accuracy']     + hist2.history['accuracy']  \n    val_acc  = hist1.history['val_accuracy'] + hist2.history['val_accuracy']\n\n    loss     = hist1.history['loss']         + hist2.history['loss']\n    val_loss = hist1.history['val_loss']     + hist2.history['val_loss']\n    #initial_epochs = initial_epochs\n    plt.figure(figsize=(8, 8))\n    plt.subplot(2, 1, 1)\n    plt.plot(acc, label='Training Accuracy')\n    plt.plot(val_acc, label='Validation Accuracy')\n    #plt.ylim([0.8, 1])\n    plt.plot([initial_epochs-1,initial_epochs-1],\n              plt.ylim(), label='Start Fine Tuning')\n    plt.legend(loc='lower right')\n    plt.ylabel('Accuracy')\n    plt.ylim([min(plt.ylim()),1])\n    plt.title('Training and Validation Accuracy')\n\n    plt.subplot(2, 1, 2)\n    plt.plot(loss, label='Training Loss')\n    plt.plot(val_loss, label='Validation Loss')\n    plt.plot([initial_epochs-1,initial_epochs-1],\n             plt.ylim(), label='Start Fine Tuning')\n    plt.legend(loc='upper right')\n    plt.ylabel('Cross Entropy')\n    #plt.ylim([0,1.0])\n    #plt.ylim([min(plt.ylim()),1])\n    plt.title('Training and Validation Loss')\n    plt.xlabel('epoch')\n    plt.show()\n    \nPolt_history_fineTune(hist, hist_2)\nplt.savefig('model/hist_3.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing model 1"},{"metadata":{},"cell_type":"markdown","source":"### * Testing model 1: on 25000 image "},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing known data in train folder\nTest_Model_known_Data(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### * Testing model 1: on 12500 image (test data)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing unknown data in test folder\npredict,test_df =Model_predict(model,test_directory)\nSave_Submission(predict,model,\"model\",test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot sample of predicted result"},{"metadata":{"trusted":true},"cell_type":"code","source":"Plot_predict(predict,test_directory,test_df)\nplt.savefig('model/predicted.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  Model 2 "},{"metadata":{"trusted":true},"cell_type":"code","source":"if not os.path.exists('model2'):\n    os.makedirs(\"model2\")   \n# build the InceptionV3 network\nbase_model2 = applications.InceptionV3(input_shape=IMG_SHAPE,\n                                       weights='imagenet', \n                                       include_top=False) #, pooling='average\nprint(\"base_model.layers= \", len(base_model2.layers)) #155\n\n#Feature extraction\n#Freeze the convolutional base\n#for layer in base_model2.layers[:100]:\n#    layer.trainable = False\nfor layer in base_model2.layers:\n    layer.trainable = True    \n# build a classifier model to put on top of the convolutional model\ntop_model2 = Sequential()\ntop_model2.add(GlobalAveragePooling2D())\ntop_model2.add(Dense(1, activation='sigmoid'))\n\nmodel2 = Sequential()\nmodel2.add(base_model2)\nmodel2.add(top_model2)\n\nmodel2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model2_train(learningRate, epochs, itration):\n    model2.compile(loss='binary_crossentropy',  #categorical_crossentropy\n                  optimizer=optimizers.RMSprop(lr=learningRate),\n                  #optimizer=optimizers.SGD(lr=learningRate, momentum=0.9),\n                  metrics=['accuracy'])\n    \n    save = 'model2/model2.weights.best_InceptionV3_' + itration + '.hdf5'\n    checkpointer = ModelCheckpoint(filepath= save , \n                                   verbose=1, save_best_only=True)\n\n    hist2 = model2.fit_generator(\n            train_generator,\n            samples_per_epoch=nb_train_samples,\n            epochs=epochs,\n            validation_data=validation_generator,\n            validation_steps=nb_validation_samples // batch_size,\n            callbacks=[checkpointer])\n    return hist2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 6\nlearningRate=1e-4\nhist2 = model2_train(learningRate, epochs, '1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save neural network structure and weights\nmodel2_structure = model2.to_json()\nf = Path(\"model2/model2_structure_InceptionV3.json\")\nf.write_text(model2_structure)\nmodel2.save_weights(\"model2/model2_weights_InceptionV3_1.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Polt_history(hist2)\nplt.savefig('model2/hist2.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fine tune model 2"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load neural network structure and weights\n#model2.load_weights(\"model2/model2_weights_ResNet50.h5\")\nepochs = 5\nlearningRate=1e-5\nhist2_2 = model2_train(learningRate, epochs, '2')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.save_weights(\"model2/model2_weights_InceptionV3_2.h5\")\nPolt_history(hist2_2)\nplt.savefig('model2/hist2_2.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Testing model 2"},{"metadata":{},"cell_type":"markdown","source":"* Testing model 2: on 25000 image "},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing known data in train folder\nTest_Model_known_Data(model2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* Testing model 2: on 12500 image (test data)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#testing unknown data in test folder\npredict2,test_df =Model_predict(model2,test_directory)\nSave_Submission(predict2,model2,\"model2\",test_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot sample of predicted result"},{"metadata":{"trusted":true},"cell_type":"code","source":"Plot_predict(predict2,test_directory,test_df)\nplt.savefig('model2/predicted2.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#remove test folder\nif  os.path.exists('test1'):\n    shutil.rmtree(\"test1\") \nif  os.path.exists('data'):\n    shutil.rmtree(\"data\")\nif  os.path.exists('Alldata'):\n    shutil.rmtree(\"Alldata\") \n    \nfile1 = \"model/model.weights.best_InceptionV3_1.hdf5\"\nfile2 = \"model/model.weights.best_InceptionV3_2.hdf5\"\nfile3 = \"model/model_weights_InceptionV3_1.h5\"\nfile4 = \"model2/model2.weights.best_InceptionV3_1.hdf5\"\nfile5 = \"model2/model2.weights.best_InceptionV3_2.hdf5\"\nfile6 = \"model2/model2_weights_InceptionV3_1.h5\"\n\nif  os.path.isfile(file1):\n    os.remove(file1)    \nif  os.path.isfile(file2):\n    os.remove(file2) \nif  os.path.isfile(file3):\n    os.remove(file3) \nif  os.path.isfile(file4):\n    os.remove(file4) \nif  os.path.isfile(file5):\n    os.remove(file5) \nif  os.path.isfile(file6):\n    os.remove(file6) ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}