{"nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "version": "3.6.4", "mimetype": "text/x-python", "file_extension": ".py", "pygments_lexer": "ipython3", "name": "python"}}, "cells": [{"metadata": {"_uuid": "71b0fafaadbd398615ef64181e3ea4587cc1a0da", "_cell_guid": "93490959-744b-4e23-ac3b-775e5460da66"}, "cell_type": "markdown", "source": ["# Exploration of Data Science Bowl 2018 Data\n", "---\n", "This short notebook will examine the images/masks in both training and testing sets.\n", "\n", "### Table of Contents\n", "1. [Load Data](#Load_Data)\n", "2. [Helper Functions](#Helper Functions)\n", "3. [Data Exploration](#Data_Exploration)"]}, {"metadata": {"_uuid": "606cb4f2f008ce4fc75d50091b3ef2fb94f2db9a", "_cell_guid": "b0e87017-5228-490e-895d-f1f5071805c2"}, "cell_type": "markdown", "source": ["<a id='Load_Data'></a>\n", "## Load Data\n", "---"]}, {"metadata": {"_uuid": "f7da6de2778ba900becb91b28df131aa2da42a84", "_cell_guid": "57e1a37d-b67d-4003-81ff-d4d70c339bbe", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import os\n", "import shutil\n", "import cv2\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "%matplotlib inline\n", "\n", "print(\"Pandas Version:\", pd.__version__)\n", "print(\"Numpy Version:\", np.__version__)\n", "print(\"OpenCV Version:\", cv2.__version__)"], "outputs": []}, {"metadata": {"_uuid": "a69b4a4176b2d52a92b60972ec22781e9a2f41ff", "_cell_guid": "5e4d9a7d-0965-4471-a033-58d4a2887096", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["base_dir = \"../input\"\n", "\n", "# Training dir/ids/imgs\n", "train_dir = os.path.join(base_dir, \"stage1_train\")\n", "train_ids = [path for path in os.listdir(train_dir)]\n", "# Dictionary mappings from IDs to images and masks\n", "train_imgs = dict([(ID, os.listdir(os.path.join(train_dir, ID, \"images\"))[0]) for ID in train_ids])\n", "train_masks = dict([(ID, os.listdir(os.path.join(train_dir, ID, \"masks\"))) for ID in train_ids])\n", "msk_cnt = 0\n", "for msk in train_masks.values():\n", "    msk_cnt += len(msk)\n", "\n", "# Testing dir/ids/imgs\n", "test_dir = os.path.join(base_dir, \"stage1_test\")\n", "test_ids = [path for path in os.listdir(test_dir)]\n", "# Dictionary mappings from IDs to images\n", "test_imgs = dict([(ID, os.listdir(os.path.join(test_dir, ID, \"images\"))[0]) for ID in test_ids])\n", "    \n", "print(\"Number of train ID files:\", len(train_ids))\n", "print(\"Number of train images:\", len(train_imgs))\n", "print(\"Number of train masks:\", msk_cnt)\n", "print()\n", "print(\"Number of test ID files:\", len(test_ids))\n", "print(\"Number of test images:\", len(test_imgs))"], "outputs": []}, {"metadata": {"_uuid": "d18c3966b97190f9beb1a4291746b2edb6e9b270", "_cell_guid": "f1438b0f-df89-47b3-97ea-161861079c18"}, "cell_type": "markdown", "source": ["**Train Dataset:** 1 image per ID, multiple masks per ID.\n", "\n", "** Test Dataset:** 1 image per ID."]}, {"metadata": {"_uuid": "eee0cbef45925ca156e19875555924cf38a6b985", "_cell_guid": "0326b0a3-1f8c-488d-ada2-76cb742b1c34"}, "cell_type": "markdown", "source": ["<a id='Helper Functions'></a>\n", "## Helper Functions\n", "---"]}, {"metadata": {"_uuid": "dd39db63abfc6c23c62009b0318948b028c0962f", "_cell_guid": "91c34c63-38bc-4577-aa4d-f0fedec61eb4", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["def load_img_shapes(path_to_img):\n", "    return cv2.imread(path_to_img).shape"], "outputs": []}, {"metadata": {"_uuid": "84af483424bd8e56a5aabe2aa41e4f135ed8e9aa", "_cell_guid": "472bbcf2-60a2-4bea-9389-015ae730086b", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["def load_img(path_to_img):\n", "    img = cv2.imread(path_to_img)\n", "    return img"], "outputs": []}, {"metadata": {"_uuid": "56697f1c41bebf6736a67a88bc7cede9dbe1c82f", "_cell_guid": "79a6b836-70e6-461a-b321-422534395e90"}, "cell_type": "markdown", "source": ["<a id='Data_Exploration'></a>\n", "## Data Exploration\n", "---\n", "Let's visualize some images and their masks."]}, {"metadata": {"_uuid": "d4ff8e90cc5e54f94c829f59575e2df697d8b7d5", "_cell_guid": "7f001ada-7401-4a2a-90c4-fd27726dcb54", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["# Grab 1 example image and 8 example masks for that image\n", "sample_img_id = train_ids[0]\n", "sample_img_path = os.path.join(train_dir, sample_img_id, \"images\", train_imgs[sample_img_id])\n", "sample_msk_paths = os.listdir(os.path.join(train_dir, sample_img_id, \"masks\"))[:8]\n", "\n", "# Load image\n", "img = load_img(sample_img_path)\n", "\n", "# Plot image\n", "plt.imshow(img)\n", "plt.title(train_imgs[sample_img_id])\n", "\n", "# Plot masks\n", "plt.figure(figsize=(17, 10))\n", "rows = 4\n", "img_per_row = len(sample_msk_paths) // rows\n", "for i in range(len(sample_msk_paths)):\n", "    mask = load_img(os.path.join(train_dir, sample_img_id, \"masks\", sample_msk_paths[i]))\n", "    plt.subplot(rows, img_per_row, i+1)\n", "    plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)\n", "    plt.imshow(mask)\n", "    plt.title(\"Mask_\" + str(i+1))\n", "plt.show()\n", "\n", "# Print number of image masks and mask names\n", "print(len(sample_msk_paths), \"Masks\")\n", "print(\"Mask image names: \", sample_msk_paths)"], "outputs": []}, {"metadata": {"_uuid": "a1cb68b47e7cd44fc3c4948e095a6d78043e6bd6", "_cell_guid": "6224d557-a59d-48a4-b954-0ab02adec817"}, "cell_type": "markdown", "source": ["Now let's plot distribution of image sizes in both datasets."]}, {"metadata": {"_uuid": "3de3a4ae95429432bf3025257a2850e4adeb4d8c", "_cell_guid": "344eb968-62d1-43c6-bf55-85d863fc03d7", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["# Load distribution of training/testing image sizes\n", "train_shapes = []\n", "test_shapes = []\n", "for i in range(len(train_imgs)):\n", "    img_id = train_ids[i]\n", "    img_path = os.path.join(train_dir, img_id, \"images\", train_imgs[img_id])\n", "    train_shapes.append(load_img_shapes(img_path))\n", "for i in range(len(test_imgs)):\n", "    img_id = test_ids[i]\n", "    img_path = os.path.join(test_dir, img_id, \"images\", test_imgs[img_id])\n", "    test_shapes.append(load_img_shapes(img_path))\n", "\n", "df_train = pd.DataFrame({'Shapes': train_shapes})\n", "train_counts = df_train['Shapes'].value_counts()\n", "df_test = pd.DataFrame({'Shapes': test_shapes})\n", "test_counts = df_test['Shapes'].value_counts()\n", "print(\"Training Image Shapes:\")\n", "for i in range(len(train_counts)):\n", "    print(\"Shape %s counts: %d\" % (train_counts.index[i], train_counts.values[i]))\n", "print(\"*\"*50)\n", "print(\"Testing Image Shapes:\")\n", "for i in range(len(test_counts)):\n", "    print(\"Shape %s counts: %d\" % (test_counts.index[i], test_counts.values[i]))"], "outputs": []}, {"metadata": {"_uuid": "51a5e9d7bcf02335127e5ba2bad919c031e562d6", "_cell_guid": "c509ce64-e6d4-4170-97ff-f6a173bdc296", "collapsed": true}, "cell_type": "code", "execution_count": null, "source": ["# Plot distribution of train/test image shapes\n", "plt.figure(figsize=(14, 10))\n", "sns.barplot(x=train_counts.index, y=train_counts.values)\n", "plt.title(\"Train Dataset\")\n", "\n", "plt.figure(figsize=(14, 10))\n", "sns.barplot(x=test_counts.index, y=test_counts.values)\n", "plt.title(\"Test Dataset\")\n", "\n", "plt.show()"], "outputs": []}, {"metadata": {"_uuid": "2558e1f8ffc32598785697eff8f2b235c39cd118", "_cell_guid": "70fd17d7-fc2f-4d18-bfb3-007aa45dad52"}, "cell_type": "markdown", "source": ["We can see that most images have a shape of (256, 256, 3). All images have 3 color channels (RGB). Only 1 training image has a large size of 1040 by 1388 by 3 pixels. Because a lot of images have different sizes we should resize them to be 256 by 256 by 3 pixels since that is the majority shape.\n", "\n", "Thanks for reading!"]}], "nbformat_minor": 1}