{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook takes \"dogs-vs-cats-redux-kernels-edition\" data and saves them as an output with the following structure \n\nunder /kaggle/output/working/:\n\n* .../train_validate (a directory with images for training and validation)\n    * .../cats (a sub-directory with 12500 cats images)\n    * .../dogs (a sub-directory with 12500 dogs images)\n* .../test_dir/test (a directory with 12500 unlabelled images for testing)\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Unzip raw data"},{"metadata":{},"cell_type":"markdown","source":"Unzip \"train.zip\" to \"/kaggle/working/train\" "},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\n# train data\nlocal_zip = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/train.zip'\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall()\nzip_ref.close()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Duplicate test data by unzipping it to \"/kaggle/working/test_dir/test\" (a directory structure needed to be able to load test images using ```tf.keras.preprocessing.image_dataset_from_directory```)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\n# test data\nlocal_zip = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall()\nzip_ref.close()\n\n!cd '/kaggle/working'\n!mkdir test_dir\n!cd test_dir\n!mv test test_dir","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nif len(os.listdir('/kaggle/working/test_dir/test'))==12500:\n    print(\"Test images are saved into {}\".format('/kaggle/working/test_dir/test'))\nelse:\n    raise ValueError(\"The number of test images in '/kaggle/working/test_dir/test' is not 12500\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Unzip \"test.zip\" to \"/kaggle/working/test\" (a directory structure needed to be able to load test images using ```tensorflow.keras.preprocessing.image.ImageDataGenerator.flow_from_dataframe```)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\n# test data\nlocal_zip = '/kaggle/input/dogs-vs-cats-redux-kernels-edition/test.zip'\nzip_ref = zipfile.ZipFile(local_zip, 'r')\nzip_ref.extractall()\nzip_ref.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nif len(os.listdir('/kaggle/working/test'))==12500:\n    print(\"Test images are saved into {}\".format('/kaggle/working/test'))\nelse:\n    raise ValueError(\"The number of test images in '/kaggle/working/test' is not 12500\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Initialize a folder with subfolders for cats and dogs needed to keep the data from the \"train\" folder."},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\ntry:\n  #set the base directory to be the project directory\n    base_dir = '/kaggle/working/'\n    #initialize a directory with subdirectories for cats and dogs\n    for sub_directory in ('train_validate/cats','train_validate/dogs'):\n        path = os.path.join(base_dir, sub_directory)\n        os.makedirs(path,exist_ok=True)\nexcept OSError:\n    pass","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sort the images from the \"train\" folder into cats (\"train_validate/cats\") and dogs (\"train_validate/dogs\")."},{"metadata":{"trusted":true},"cell_type":"code","source":"from random import seed\nfrom random import random\nfrom shutil import copyfile\nimport os,sys\n\n# set the directories according to the structure defined before\n#base_dir = '/kaggle/working/'\nsource_dir = '/kaggle/working/train/'\ndst_dir = '/kaggle/working/train_validate/'\n\n#seed=1 to make sure we get the same split each time the code is run\nseed(1)\ncounter = 0\n#\nfor file in os.listdir(source_dir):\n  src = source_dir + file\n  # select cats' images\n  if file.startswith('cat'):\n    dst = dst_dir + 'cats/'  + file\n    copyfile(src, dst)\n  elif file.startswith('dog'):\n    dst = dst_dir + 'dogs/'  + file\n    copyfile(src, dst)\n  #\n  # print progress every 10th iteration\n  counter +=1 \n  if counter%100==0:\n    ii=(counter/len(os.listdir(source_dir))*100)\n    print(\"\\r Progress: {} %\".format(ii), end=\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nif len(os.listdir('/kaggle/working/train_validate/cats'))==12500:\n    print(\"Cats images for training and validation are saved into {}\".format(dst_dir+'cats'))\nelse:\n    raise ValueError(\"The number of cats images read is not 12500\")\nif len(os.listdir('/kaggle/working/train_validate/cats'))==12500:\n    print(\"Dogs images for training and validation are saved into {}\".format(dst_dir+'dogs'))\nelse:\n    raise ValueError(\"The number of dogs images read is not 12500\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nprint(\"Number of images in cats folder :\", len(os.listdir(\"/kaggle/working/train_validate/cats\")))\nprint(\"Number of images in dogs folder :\", len(os.listdir(\"/kaggle/working/train_validate/dogs\")))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Notebook's output:\n\n* \"train_validate\" folder contains images for training and validation. They are split into two classes (cats and dogs). But they are not split for training and validation yet.\n\n* \"test\" folder contains test images we will use later for testing (directory structure needed to load data using ```tensorflow.keras.preprocessing.image.ImageDataGenerator.flow_from_dataframe```)\n\n* \"test_dir/test\" folder contains test images we will use later for testing (directory structure needed to load data using ```tf.keras.preprocessing.image_dataset_from_directory```)"},{"metadata":{},"cell_type":"markdown","source":"* \"train\" folder contains images which we already copied to \"train_validate\" folder and can be deleted. "},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf /kaggle/working/train/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nif os.path.exists(\"/kaggle/working/train/\")==False:\n    print(\" The 'train' folder {}\".format(source_dir), \" was removed\")\nelse:\n    raise ValueError(\"train folder was not removed\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}