{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Some tricks to combat overfitting**\n\n* Part 1 [Intro to CNN (Dogs vs. Cats)](https://www.kaggle.com/imcr00z/intro-to-cnn-dogs-vs-cats)\n* Part 2\n* Part 3 [Transfer Learning (Dogs vs. Cats) 98% acc.](https://www.kaggle.com/imcr00z/transfer-learning-dogs-vs-cats-98-acc)\n\nIn this part i use augmentation and dropout layers to reduce overfitting and increase the predictive power of the model.","metadata":{}},{"cell_type":"code","source":"import os\nimport tqdm\nimport shutil\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\n\nimport logging\nlogger = tf.get_logger()\nlogger.setLevel(logging.ERROR)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:53:17.303517Z","iopub.execute_input":"2021-11-02T14:53:17.30386Z","iopub.status.idle":"2021-11-02T14:53:19.540779Z","shell.execute_reply.started":"2021-11-02T14:53:17.303805Z","shell.execute_reply":"2021-11-02T14:53:19.539872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-02T14:53:19.546695Z","iopub.execute_input":"2021-11-02T14:53:19.547017Z","iopub.status.idle":"2021-11-02T14:53:19.555055Z","shell.execute_reply.started":"2021-11-02T14:53:19.546974Z","shell.execute_reply":"2021-11-02T14:53:19.554449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data\nCopied from my [Intro to CNN (Dogs vs. Cats)](http://https://www.kaggle.com/imcr00z/intro-to-cnn-dogs-vs-cats)","metadata":{}},{"cell_type":"code","source":"# !rm -rf /kaggle/content","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:53:24.385131Z","iopub.execute_input":"2021-11-02T14:53:24.385939Z","iopub.status.idle":"2021-11-02T14:53:24.390092Z","shell.execute_reply.started":"2021-11-02T14:53:24.38589Z","shell.execute_reply":"2021-11-02T14:53:24.389271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONTENT_DIR = '/kaggle/content'\n\nTRAIN_DIR = CONTENT_DIR + '/train'\nTRAIN_DIR_DOG = TRAIN_DIR + '/dog'\nTRAIN_DIR_CAT = TRAIN_DIR + '/cat'\n\nVALID_DIR = CONTENT_DIR + '/valid'\nVALID_DIR_DOG = VALID_DIR + '/dog'\nVALID_DIR_CAT = VALID_DIR + '/cat'\n\nif not os.path.exists(CONTENT_DIR):\n    import zipfile\n    with zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip', 'r') as zipf:\n        zipf.extractall(CONTENT_DIR)\n\n    # Split cats and dogs images to train and valid datasets\n    img_filenames = os.listdir(TRAIN_DIR)\n    print('Num of images:', len(img_filenames))\n\n    dog_filenames = [fn for fn in img_filenames if fn.startswith('dog')]\n    cat_filenames = [fn for fn in img_filenames if fn.startswith('cat')]\n\n    dataset_filenames = train_test_split(\n        dog_filenames, cat_filenames, test_size=0.1, shuffle=True, random_state=42\n    )\n\n    train_dog_total, valid_dog_total, train_cat_total, valid_cat_total = [len(fns) for fns in dataset_filenames]\n    train_total = train_dog_total + train_cat_total\n    valid_total = valid_dog_total + valid_cat_total\n    print('Train: {}, test: {}'.format(train_total, valid_total))\n\n    # Move images\n    make_dirs = [TRAIN_DIR_DOG, VALID_DIR_DOG, TRAIN_DIR_CAT, VALID_DIR_CAT]\n    for dir, fns in zip(make_dirs, dataset_filenames):\n        os.makedirs(dir, exist_ok=True)\n        for fn in tqdm.tqdm(fns):\n            shutil.move(os.path.join(TRAIN_DIR, fn), dir)\n        print('elements in {}: {}'.format(dir, len(os.listdir(dir))))","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:53:24.931221Z","iopub.execute_input":"2021-11-02T14:53:24.932287Z","iopub.status.idle":"2021-11-02T14:53:34.015103Z","shell.execute_reply.started":"2021-11-02T14:53:24.932238Z","shell.execute_reply":"2021-11-02T14:53:34.014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Augmentation\nWe change images to increase the size of the dataset.","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:20:22.774343Z","iopub.execute_input":"2021-11-02T14:20:22.775535Z","iopub.status.idle":"2021-11-02T14:20:22.78009Z","shell.execute_reply.started":"2021-11-02T14:20:22.775482Z","shell.execute_reply":"2021-11-02T14:20:22.779382Z"}}},{"cell_type":"code","source":"BATCH_SIZE = 128\nIMAGE_SHAPE = 128","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:53:36.033199Z","iopub.execute_input":"2021-11-02T14:53:36.033513Z","iopub.status.idle":"2021-11-02T14:53:36.039373Z","shell.execute_reply.started":"2021-11-02T14:53:36.033481Z","shell.execute_reply":"2021-11-02T14:53:36.038529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_images(images):\n    fig, axes = plt.subplots(1, 5, figsize=(20, 20))\n    for img, ax in zip(images, axes.flatten()):\n        ax.imshow(img)\n    plt.tight_layout()\n    plt.show()    ","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:53:38.149693Z","iopub.execute_input":"2021-11-02T14:53:38.150463Z","iopub.status.idle":"2021-11-02T14:53:38.156903Z","shell.execute_reply.started":"2021-11-02T14:53:38.150404Z","shell.execute_reply":"2021-11-02T14:53:38.156118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_augmentation(generator):\n    image_data = image_generator.flow_from_directory(\n        directory=TRAIN_DIR,\n        target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n        batch_size=BATCH_SIZE,\n        shuffle=True\n    )\n    augmented_images = [image_data[0][0][0] for _ in range(5)]\n    plot_images(augmented_images)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:54:05.495968Z","iopub.execute_input":"2021-11-02T14:54:05.496499Z","iopub.status.idle":"2021-11-02T14:54:05.502648Z","shell.execute_reply.started":"2021-11-02T14:54:05.49645Z","shell.execute_reply":"2021-11-02T14:54:05.501813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Horisontal flipping","metadata":{"execution":{"iopub.status.busy":"2021-11-02T13:43:14.932267Z","iopub.execute_input":"2021-11-02T13:43:14.932708Z","iopub.status.idle":"2021-11-02T13:43:14.937507Z","shell.execute_reply.started":"2021-11-02T13:43:14.932672Z","shell.execute_reply":"2021-11-02T13:43:14.936267Z"}}},{"cell_type":"code","source":"image_generator = ImageDataGenerator(rescale=1./255, horizontal_flip=True)\nshow_augmentation(image_generator)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:54:09.41247Z","iopub.execute_input":"2021-11-02T14:54:09.412792Z","iopub.status.idle":"2021-11-02T14:54:12.608517Z","shell.execute_reply.started":"2021-11-02T14:54:09.41276Z","shell.execute_reply":"2021-11-02T14:54:12.597747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rotation","metadata":{"execution":{"iopub.status.busy":"2021-11-02T13:49:10.713635Z","iopub.execute_input":"2021-11-02T13:49:10.714013Z","iopub.status.idle":"2021-11-02T13:49:10.717742Z","shell.execute_reply.started":"2021-11-02T13:49:10.713981Z","shell.execute_reply":"2021-11-02T13:49:10.717095Z"}}},{"cell_type":"code","source":"image_generator = ImageDataGenerator(rescale=1./255, rotation_range=45)\nshow_augmentation(image_generator)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:54:14.740482Z","iopub.execute_input":"2021-11-02T14:54:14.740809Z","iopub.status.idle":"2021-11-02T14:54:20.252621Z","shell.execute_reply.started":"2021-11-02T14:54:14.740776Z","shell.execute_reply":"2021-11-02T14:54:20.251654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Zoom","metadata":{"execution":{"iopub.status.busy":"2021-11-02T13:50:59.560363Z","iopub.execute_input":"2021-11-02T13:50:59.560702Z","iopub.status.idle":"2021-11-02T13:50:59.564684Z","shell.execute_reply.started":"2021-11-02T13:50:59.560658Z","shell.execute_reply":"2021-11-02T13:50:59.563919Z"}}},{"cell_type":"code","source":"image_generator = ImageDataGenerator(rescale=1./255, zoom_range=0.5)\nshow_augmentation(image_generator)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:54:24.774819Z","iopub.execute_input":"2021-11-02T14:54:24.775608Z","iopub.status.idle":"2021-11-02T14:54:30.023591Z","shell.execute_reply.started":"2021-11-02T14:54:24.775565Z","shell.execute_reply":"2021-11-02T14:54:30.022605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Putting it all together","metadata":{"execution":{"iopub.status.busy":"2021-11-02T13:56:47.267956Z","iopub.execute_input":"2021-11-02T13:56:47.268668Z","iopub.status.idle":"2021-11-02T13:56:47.273083Z","shell.execute_reply.started":"2021-11-02T13:56:47.268623Z","shell.execute_reply":"2021-11-02T13:56:47.272115Z"}}},{"cell_type":"code","source":"image_generator = ImageDataGenerator(\n    rotation_range=40,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    shear_range=0.2,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    rescale=1./255\n)\ntrain_data = image_generator.flow_from_directory(\n    directory=TRAIN_DIR,\n    target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n    batch_size=BATCH_SIZE,\n    shuffle=True,\n    class_mode='binary'\n)\nshow_augmentation(image_generator)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:56:53.479636Z","iopub.execute_input":"2021-11-02T14:56:53.479924Z","iopub.status.idle":"2021-11-02T14:56:59.393331Z","shell.execute_reply.started":"2021-11-02T14:56:53.479894Z","shell.execute_reply":"2021-11-02T14:56:59.392688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Valid data","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:44:56.437248Z","iopub.execute_input":"2021-11-02T14:44:56.438089Z","iopub.status.idle":"2021-11-02T14:44:56.443582Z","shell.execute_reply.started":"2021-11-02T14:44:56.438028Z","shell.execute_reply":"2021-11-02T14:44:56.442265Z"}}},{"cell_type":"code","source":"valid_generator = ImageDataGenerator(rescale=1./255)\nvalid_data = valid_generator.flow_from_directory(\n    directory=VALID_DIR,\n    target_size=(IMAGE_SHAPE, IMAGE_SHAPE),\n    batch_size=BATCH_SIZE,\n    class_mode='binary',\n    shuffle=False\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:10.55962Z","iopub.execute_input":"2021-11-02T14:57:10.560281Z","iopub.status.idle":"2021-11-02T14:57:10.673238Z","shell.execute_reply.started":"2021-11-02T14:57:10.560219Z","shell.execute_reply":"2021-11-02T14:57:10.672261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model\nWith dropout layer.","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:14:47.199327Z","iopub.execute_input":"2021-11-02T14:14:47.200969Z","iopub.status.idle":"2021-11-02T14:14:47.207381Z","shell.execute_reply.started":"2021-11-02T14:14:47.200876Z","shell.execute_reply":"2021-11-02T14:14:47.206221Z"}}},{"cell_type":"code","source":"model = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(\n        filters=32,\n        kernel_size=(3, 3),\n        activation='relu',\n        input_shape=(IMAGE_SHAPE, IMAGE_SHAPE, 3)\n    ),\n    tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),\n\n    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n\n    tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n\n    tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),\n    tf.keras.layers.MaxPooling2D(2, 2),\n    \n    # dropout layer\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(units=512, activation='relu'),\n    tf.keras.layers.Dense(units=2, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:17.159975Z","iopub.execute_input":"2021-11-02T14:57:17.160922Z","iopub.status.idle":"2021-11-02T14:57:17.272998Z","shell.execute_reply.started":"2021-11-02T14:57:17.160859Z","shell.execute_reply":"2021-11-02T14:57:17.272215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:23.53873Z","iopub.execute_input":"2021-11-02T14:57:23.539589Z","iopub.status.idle":"2021-11-02T14:57:23.55193Z","shell.execute_reply.started":"2021-11-02T14:57:23.539547Z","shell.execute_reply":"2021-11-02T14:57:23.551262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:57:25.056497Z","iopub.execute_input":"2021-11-02T14:57:25.056828Z","iopub.status.idle":"2021-11-02T14:57:25.067674Z","shell.execute_reply.started":"2021-11-02T14:57:25.056788Z","shell.execute_reply":"2021-11-02T14:57:25.066819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 60\nhistory = model.fit_generator(\n    generator=train_data,\n    steps_per_epoch=(train_total + BATCH_SIZE - 1) // BATCH_SIZE,\n    epochs=EPOCHS,\n    validation_data=valid_data,\n    validation_steps=(valid_total + BATCH_SIZE - 1) // BATCH_SIZE,\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T14:58:28.753267Z","iopub.execute_input":"2021-11-02T14:58:28.753666Z","iopub.status.idle":"2021-11-02T15:30:19.979024Z","shell.execute_reply.started":"2021-11-02T14:58:28.753624Z","shell.execute_reply":"2021-11-02T15:30:19.977661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train visualization\nLook at it and compare with the graph [here](http://https://www.kaggle.com/imcr00z/intro-to-cnn-dogs-vs-cats)","metadata":{"execution":{"iopub.status.busy":"2021-11-02T15:30:19.981628Z","iopub.execute_input":"2021-11-02T15:30:19.981901Z","iopub.status.idle":"2021-11-02T15:30:19.986136Z","shell.execute_reply.started":"2021-11-02T15:30:19.981871Z","shell.execute_reply":"2021-11-02T15:30:19.985591Z"}}},{"cell_type":"code","source":"plt.figure(figsize=(12, 8))\n\nplt.subplot(1, 2, 1)\nplt.plot(range(EPOCHS), history.history['accuracy'], label='train')\nplt.plot(range(EPOCHS), history.history['val_accuracy'], label='valid')\nplt.legend(loc='lower right')\nplt.title('Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(range(EPOCHS), history.history['loss'], label='train')\nplt.plot(range(EPOCHS), history.history['val_loss'], label='valid')\nplt.legend(loc='upper right')\nplt.title('Loss (sparse_categorical_crossentropy)')\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}