{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Extracting zip files"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))\n\nimport zipfile\n\nwith zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as z:\n    z.extractall(\".\")\n    \nwith zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\",\"r\") as z:\n    z.extractall(\".\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport cv2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining train path"},{"metadata":{"trusted":true},"cell_type":"code","source":"main_dir = \"/kaggle/working/\"\ntrain_dir = \"train\"\npath = os.path.join(main_dir,train_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dir = \"test1\"\ntest_path = os.path.join(main_dir, test_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# function for labels\ndef label(img):\n    word = img.split('.')[0]\n    if word == 'cat':\n        return 1\n    elif word == 'dog':\n        return 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\nimport cv2\nimport os\nimport numpy as np\nfrom random import shuffle\n\nIMG_SIZE = 50\n\ntrain_data = []\ntrain_labels = []\ndef create_train_data(path):\n    for img in os.listdir(path):\n        labels = label(img)\n        path2 = os.path.join(path, img)\n        img = cv2.imread(path2, cv2.IMREAD_GRAYSCALE)\n        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))\n        train_data.append(img)\n        train_labels.append(labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining X and y"},{"metadata":{"trusted":true},"cell_type":"code","source":"create_train_data(path)\nX = np.array(train_data).reshape(-1, 50,50,1)\ny = np.array(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X/255.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TF libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Activation, Conv2D, MaxPooling2D, BatchNormalization","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model1 = Sequential()\n\nmodel1.add(Conv2D(32,(3,3), activation = 'relu', input_shape = X.shape[1:]))\nmodel1.add(BatchNormalization())\nmodel1.add(MaxPooling2D(pool_size = (2,2)))\nmodel1.add(Dropout(0.25))\n\nmodel1.add(Conv2D(64,(3,3), activation = 'relu'))\nmodel1.add(BatchNormalization())\nmodel1.add(MaxPooling2D(pool_size = (2,2)))\nmodel1.add(Dropout(0.25))\n\n\nmodel1.add(Flatten())\nmodel1.add(Dense(64, activation = 'relu'))\nmodel1.add(Dense(32, activation = 'relu'))\n\nmodel1.add(Dense(1, activation = 'sigmoid'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model1.compile(\n    optimizer = \"adam\",\n    loss = \"binary_crossentropy\",\n    metrics = [\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wandb for tracking model's performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%capture\n!pip install wandb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing Wandb"},{"metadata":{"trusted":true},"cell_type":"code","source":"import wandb\nfrom wandb.keras import WandbCallback\n\nwandb.login()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Initialising Wandb"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nwandb.init(entity='pratikraut_', project='cats-vs-dogs')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitting model"},{"metadata":{"trusted":true},"cell_type":"code","source":"my_callbacks = WandbCallback()\nmodel1.fit(X, y, epochs=20, batch_size=32, validation_split=0.2, callbacks = my_callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing test data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = []\nidd = []\ndef process_test_data(test_path):\n    for img in os.listdir(test_path):\n        tpath = os.path.join(test_path, img)\n        img_num = img.split('.')[0]\n        img = cv2.imread(tpath, cv2.IMREAD_GRAYSCALE)\n        img = cv2.resize(img, (32, 32))\n        test_data.append(np.array(img))\n        idd.append(img_num)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"process_test_data(test_path)\nX_test = np.array(test_data).reshape(-1,50,50,1)\nX_test = X_test/255","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predicting on test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model1.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluating model"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss, accuracy = model1.evaluate(X_test)\nprint('Test Error Rate: ', round((1-accuracy)*100, 2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining 2nd model with different units"},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = Sequential()\n\nmodel2.add(Conv2D(64,(3,3), activation = 'relu', input_shape = X.shape[1:]))\nmodel2.add(BatchNormalization())\nmodel2.add(MaxPooling2D(pool_size = (2,2)))\nmodel2.add(Dropout(0.25))\n\nmodel2.add(Conv2D(128,(3,3), activation = 'relu'))\nmodel2.add(BatchNormalization())\nmodel2.add(MaxPooling2D(pool_size = (2,2)))\nmodel2.add(Dropout(0.25))\n\n\nmodel2.add(Flatten())\nmodel2.add(Dense(512, activation = 'relu'))\nmodel2.add(BatchNormalization())\nmodel2.add(Dropout(0.5))\nmodel2.add(Dense(2, activation = 'softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model2.compile(\n    optimizer = \"adam\",\n    loss = \"binary_crossentropy\",\n    metrics = [\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"my_callbacks = WandbCallback()\nmodel2.fit(X, y, epochs=20, batch_size=64, validation_split=0.2, callbacks = my_callbacks)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing data for new model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nfilenames = os.listdir(\"/kaggle/working/train\")\ncategories = []\nfor filename in filenames:\n    category = filename.split('.')[0]\n    if category == 'dog':\n        categories.append(str(1))\n    else:\n        categories.append(str(0))\n\ndf = pd.DataFrame({\n    'filename': filenames,\n    'category': categories\n})\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['category'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining model - VGG16"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\nfrom keras.layers import Conv2D, MaxPooling2D, Dropout, Flatten, Dense, Activation,GlobalMaxPooling2D\nfrom keras import applications\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras import optimizers\nfrom keras.applications import VGG16\nfrom keras.models import Model\n\nimage_size = 224\ninput_shape = (image_size, image_size, 3)\n\nepochs = 5\nbatch_size = 16\n\npre_trained_model = VGG16(input_shape=input_shape, include_top=False, weights=\"imagenet\")\n\n    \nfor layer in pre_trained_model.layers[:15]:\n    layer.trainable = False\n\nfor layer in pre_trained_model.layers[15:]:\n    layer.trainable = True\n    \nlast_layer = pre_trained_model.get_layer('block5_pool')\nlast_output = last_layer.output\n    \n# Flatten the output layer to 1 dimension\nx = GlobalMaxPooling2D()(last_output)\n# Add a fully connected layer with 512 hidden units and ReLU activation\nx = Dense(512, activation='relu')(x)\n# Add a dropout rate of 0.5\nx = Dropout(0.5)(x)\n# Add a final sigmoid layer for classification\nx = layers.Dense(1, activation='sigmoid')(x)\n\nmodel = Model(pre_trained_model.input, x)\n\nmodel.compile(loss='binary_crossentropy',optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),\n              metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Splitting the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_df, validate_df = train_test_split(df, test_size=0.1)\ntrain_df = train_df.reset_index()\nvalidate_df = validate_df.reset_index()\n\n# validate_df = validate_df.sample(n=100).reset_index() # use for fast testing code purpose\n# train_df = train_df.sample(n=1800).reset_index() # use for fast testing code purpose\n\ntotal_train = train_df.shape[0]\ntotal_validate = validate_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Using Image data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n    rotation_range=15,\n    rescale=1./255,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest',\n    width_shift_range=0.1,\n    height_shift_range=0.1\n)\n\ntrain_generator = train_datagen.flow_from_dataframe(\n    train_df, \n    \"/kaggle/working/train\", \n    x_col='filename',\n    y_col='category',\n    class_mode='binary',\n    target_size=(image_size, image_size),\n    batch_size=batch_size\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_datagen = ImageDataGenerator(rescale=1./255)\nvalidation_generator = validation_datagen.flow_from_dataframe(\n    validate_df, \n    \"/kaggle/working/train\", \n    x_col='filename',\n    y_col='category',\n    class_mode='binary',\n    target_size=(image_size, image_size),\n    batch_size=batch_size\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Fitting the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(\n    train_generator,\n    epochs=10,\n    validation_data=validation_generator,\n    validation_steps=total_validate//batch_size,\n    steps_per_epoch=total_train//batch_size)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Saving the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import load_model \nmodel.save(\"model.h5\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Evaluating model"},{"metadata":{"trusted":true},"cell_type":"code","source":"loss, accuracy = model.evaluate_generator(validation_generator, total_validate//batch_size, workers=12)\nprint(\"Test: accuracy = %f  ;  loss = %f \" % (accuracy, loss))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model behaviour"},{"metadata":{},"cell_type":"markdown","source":"I trained 4 models for this dataset and I will share what I observed while training these models.\n\n**Model 1**\n\nModel is a simple network with Image size = 80,\n\n2 convolution layers and 2 pooling layers respectively and activation ReLu.\n\n3 Dense layers, 2 of them has ReLu activation and 1 is having sigmoid activation.\n\nEpochs = 20\n\nWith this network I acheived,\nTrain accuracy = 98.6%,\nTrain  loss = 0.036.\n\nTest accuracy = 74.4%,\nTest loss = 1.686.\n\nHere I observed that the loss was increasing with each epoch and it looked it overfitting.\nI first thought of Image size which is 80, might be having something to do with the model overfitting and increasing loss so, I then, trained the second model.\n\n**Model 2**\n\nModel is a simple network with Image size = 100,\n2 convolution layers and 2 pooling layers respectively and activation ReLu,\n3 Dense layers, 2 of them has ReLu activation and 1 is having sigmoid activation.\n\nEpochs = 20\n\nWith this network I acheived,\nTrain accuracy = 99.4%,\nTrain  loss = 0.018.\n\nTest accuracy = 77.2%,\nTest loss = 1.55.\n\nThe loss is still increasing even if I changed the Image size so, it says that the increase in the loss and overfitting is not related to the Image sizes.\n\n**Model 3**\nKeeping the the image size = 100 as it has no relation to overfitting or loss.\nModel is a simple network with Image size = 100,\n2 convolution layers and added Batch Normalization to the convolution layers and a Dropout = 0.25\n2 pooling layers respectively and activation ReLu.\n3 Dense layers, 2 of them has ReLu activation and 1 is having sigmoid activation.\n\nEpochs = 20\n\nWith this network I acheived,\nTrain accuracy = 91.8%\nTrain  loss = 0.197\n\nTest accuracy = 76.2%\nTest loss = 0.636\n\nWith the addition of Batch Normalization and Dropout to the network the loss was not increasing anymore, there was a decrease in the loss (which means we are on the right track), the network still overfits and that is still off the track. \nWhy Batch Normalization worked ?\nWe have 12500 images of cats in our dataset and it is obvious that all the images must have different noise (one image has cat and a pillow, second image just have a cat in it so, the pillow is the noise).\nNow, everytime a batch of n size is feed into the network, it will recieve images with some difference compared to previous batches.\nSo the hidden layers will see different images everytime and it will be harder for them to learn from it.\nThis difference is called covariant shift.\nTo overcome this problem we use Batch Normalization and as we saw it worked.\nIt's like if we practise java everyday for some period then it will be easier for us to learn but if we are learning many languages then it will be harder for us to learn anything from it.\n\n\n**Model 4**\nNow, we still have the overfitting problem and I just tried to change the filters and I thought of decreasing the Image size.\n\nKeeping the the image size = 50.\n2 convolution layers and added Batch Normalization to the convolution layers and a Dropout = 0.25\n2 pooling layers respectively and activation ReLu.\n3 Dense layers, 2 of them has ReLu activation and 1 is having sigmoid activation.\n\nEpochs = 20\n\nWith this network I acheived,\nTrain accuracy = 99.07%\nTrain  loss = 0.02\n\nTest accuracy = 99.73%\nTest loss = 0.0094\n\nIt looks like the model is not overfitting the data and is better compared to Model 3.\nThen I tried to evaluate the model on the test data and I got 100% accuracy so, I think we are still in the overfitting bubble.\n\nIn the last try I tried different architecture i.e. VGG16 and it solved both the overfitting problem."},{"metadata":{},"cell_type":"markdown","source":"# You can see plots over here... \nhttps://wandb.ai/pratikraut_/cats-vs-dogs?workspace=user-pratikraut_"},{"metadata":{},"cell_type":"markdown","source":"# I'm a begineer to this and want to learn more about it, I shared my understandings and hoping to get feedbacks and corrections...."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}