{"metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"nbconvert_exporter": "python", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "file_extension": ".py", "version": "3.6.3", "name": "python", "codemirror_mode": {"version": 3, "name": "ipython"}}}, "nbformat_minor": 1, "nbformat": 4, "cells": [{"metadata": {"_cell_guid": "c60e946f-317f-4ee8-8eb7-4d15d0fd1f35", "_uuid": "9be77758ec123862b1aeca0cc0b38561698d2c35"}, "source": ["#  File Subdivision\n"], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "323f57ed-6968-4d77-b128-be843926a191", "_uuid": "9fd90b016c8e43700c8c47e187a4bf509815b4c3"}, "source": ["#### Import Dependencies"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "627d635f-d8ce-40df-ac87-14b257dd03ac", "_uuid": "1104f9b3c3fa65a3e53a44fdad2198a0d2b281b4"}, "outputs": [], "source": ["import os, sys, random, zipfile\n", "from importlib import reload  # Python 3\n", "import numpy as np\n", "import shutil\n", "from glob import glob"], "cell_type": "code", "execution_count": null}, {"metadata": {"_cell_guid": "9912aebd-22ec-4ba4-8ea3-5ccaa7d5b9f9", "_uuid": "2c2e862848dce6e0357082d15a3e664d8322f30e"}, "source": ["#### Define Constants"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "7dd85211-23e6-4d5c-b57d-aa286ac873bb", "_uuid": "547fe9a711e6160eb595af982644725843bf0d0b"}, "outputs": [], "source": ["#NOTE - notebook assumes data has been unzipped\n", "LABEL_LIST = \"breedList.txt\" #made a textfile from the list of breeds provided\n", "ID_MAP = \"labels.csv\"\n", "\n", "STORAGE_DIR = \"{input dir}\"\n", "TRAIN_DIR = STORAGE_DIR + \"train/\"\n", "VAL_DIR = STORAGE_DIR + \"valid/\"\n", "\n", "SAMPLE_DIR = STORAGE_DIR + \"sample/\"\n", "SAMPLE_TRAIN = SAMPLE_DIR + \"train/\"\n", "SAMPLE_VAL = SAMPLE_DIR + \"valid/\"\n", "\n", "RESULTS_DIR = STORAGE_DIR + \"results/\""], "cell_type": "code", "execution_count": null}, {"metadata": {"_cell_guid": "10bc518d-4980-4915-87c2-6ae8d074c3b8", "_uuid": "d4865859a4738db5f9b74cca8d2d9d565530fa59"}, "source": ["#### Define Directory Structure"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "75a6b740-a1ee-4200-a60f-96c51a91364d", "_uuid": "e31dcc5121168099be91296be8502e3def7c068f"}, "outputs": [], "source": ["%cd $STORAGE_DIR\n", "\n", "if not os.path.isdir(TRAIN_DIR):\n", "    %mkdir $TRAIN_DIR\n", "\n", "if not os.path.isdir(VAL_DIR):\n", "    %mkdir $VAL_DIR\n", "    \n", "if not os.path.isdir(SAMPLE_DIR):\n", "    %mkdir $SAMPLE_DIR\n", "    \n", "if not os.path.isdir(SAMPLE_TRAIN):\n", "    %mkdir $SAMPLE_TRAIN\n", "\n", "if not os.path.isdir(SAMPLE_VAL):\n", "    %mkdir $SAMPLE_VAL\n", "    \n", "if not os.path.isdir(RESULTS_DIR):\n", "    %mkdir $RESULTS_DIR"], "cell_type": "code", "execution_count": null}, {"metadata": {"_cell_guid": "6313dce9-f3cb-4236-8ed4-42cd907a81e1", "_uuid": "21024ffccc3240283c8153f4cc4d9c09c1523508"}, "source": ["#### Sort Data By Label"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "68ae3369-4681-4189-993b-2b57225db8cf", "_uuid": "82459262bb992df5bfc2a6c707dcc532f91195bc"}, "outputs": [], "source": ["def processLabels(labels, path):\n", "    %cd $path\n", "    for label in labels:\n", "        if not os.path.isdir(label):\n", "            %mkdir $label"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "264c5a5b-b037-4304-9f46-d76d3bdc5dfb", "_uuid": "ef4cef73c3573290ce86d1bc99c98a7537e0f250"}, "outputs": [], "source": ["def selectRandomFiles(num, path):\n", "    if len(os.listdir(path)) < num:\n", "        num = len(os.listdir(path))\n", "        \n", "    selected = []\n", "    while num >= len(selected):\n", "            choice = random.choice(os.listdir(path))\n", "            print(\"ch: \" + choice)\n", "            if choice not in selected:\n", "                selected.append(choice)\n", "    return selected"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "ff01a4ee-40bb-491f-99d5-1fd2c1d828c0", "_uuid": "91c35c3809bd1a38a3df224fac1c49ea2ecd3706"}, "outputs": [], "source": ["def createSampleSet(labels, samplePath, sourcePath, num):\n", "    processLabels(labels, samplePath)\n", "    for label in labels:\n", "        existing = len(os.listdir(samplePath + label + \"/\"))\n", "        if existing < num:\n", "            files = selectRandomFiles(num - existing, sourcePath + label + \"/\")\n", "            for file in files:\n", "                src = sourcePath + label + \"/\" + file\n", "                dest = samplePath + label + \"/\"\n", "                shutil.copy2(src, dest)"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "df4c0e3c-4066-4765-a0f3-4f3b7e6dc72c", "_uuid": "248309f81416938908b299875fc632ad756dfc0d"}, "outputs": [], "source": ["def removeUnsortedFiles(path):\n", "    files = glob(path + \"*.jpg\")\n", "    for file in files:\n", "        os.remove(file)"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "f8cf6979-03a5-496b-8a1f-1ee933ef2ee2", "_uuid": "221d94713c479e154cc891ed41e913b28be6d654"}, "outputs": [], "source": ["def createTrainingSet(labels, items):\n", "    processLabels(labels, TRAIN_DIR)\n", "    for item in items:\n", "        file = TRAIN_DIR + item[0] + \".jpg\"\n", "        dest = TRAIN_DIR + item[1] + \"/\" + item[0] + \".jpg\"\n", "        if os.path.isfile(file):\n", "            shutil.copy2(file, dest)\n", "    createSampleSet(labels, SAMPLE_TRAIN, TRAIN_DIR, 16)\n", "    removeUnsortedFiles(TRAIN_DIR)"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "599ef43b-5f64-44f6-93f3-3c743a084da7", "_uuid": "7de756d0f4433ec9dc9b7228c7c248a465ff356d"}, "outputs": [], "source": ["def createValidationSet(lables):\n", "    processLabels(labels, VAL_DIR)\n", "    for label in labels:\n", "        existing = len(os.listdir(VAL_DIR + label))\n", "        if(existing > 0):\n", "             continue\n", "        files = os.listdir(TRAIN_DIR + label)\n", "        numForValid = int(np.ceil(len(files) * .2))\n", "        for file in files[-numForValid:]:\n", "            src = TRAIN_DIR + label + \"/\" + file\n", "            dest = VAL_DIR + label + \"/\" + file\n", "            shutil.copy2(src , dest)\n", "            os.remove(src)\n", "    createSampleSet(labels, SAMPLE_VAL, VAL_DIR, 4)"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "622e0b14-f5f4-4299-a2f0-4111ad0f2d4b", "_uuid": "a6714b05e564c94a0492470e888de09de2b4fdd3"}, "outputs": [], "source": ["def sortData(labels):\n", "    items = np.loadtxt(STORAGE_DIR + ID_MAP, delimiter=',', dtype=str, skiprows=1)\n", "    createTrainingSet(labels, items)\n", "    createValidationSet(labels)"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "28b8b988-74b4-41bb-9651-b87c30925d0f", "_uuid": "eb116a2fc4f2587bb3c2ab4082b4941c99a810f2"}, "outputs": [], "source": ["def getLabels():\n", "    if os.path.isfile(STORAGE_DIR + LABEL_LIST):\n", "        try: \n", "            f = open(STORAGE_DIR + LABEL_LIST, 'r')\n", "            fileString = f.read()\n", "            return fileString.splitlines()\n", "        except IOError:\n", "            print(\"Could not read file: \", LABEL_LIST)\n", "    return []"], "cell_type": "code", "execution_count": null}, {"metadata": {"collapsed": true, "_cell_guid": "e853fd41-bfb2-469b-a78b-9b2fe7d7ffab", "_uuid": "d0bd19a7fc89c7eeadee54705d84de102c332bd8"}, "outputs": [], "source": ["labels = getLabels()\n", "sortData(labels)"], "cell_type": "code", "execution_count": null}]}