{"nbformat_minor": 1, "nbformat": 4, "cells": [{"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "190cb7a5-f6b8-471e-8951-3dc417d25b7b", "_uuid": "816e8ac3dcf679e2603a8ae3b8027d99a8dbd7f9"}, "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import os\n", "import random\n", "import cv2\n", "import numpy as np # linear algebra"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "6af43ef7-dbd4-4b86-a0c8-4c58951cad80", "_uuid": "3f970207c76ea4db203e9d384e3e53e39495d21c"}, "source": ["## constants\n", "TRAIN_DIR = \"../input/train/\"\n", "TEST_DIR = \"../input/test/\"\n", "TRAIN_SIZE = 22500\n", "TEST_SIZE = 2500\n", "DEV_RATIO = 0.1\n", "IMAGE_HEIGHT = IMAGE_WIDTH = 128\n", "OUTPUT_SIZE = 2"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "1ff26ed6-bcf4-4c0d-b656-679092098bff", "_uuid": "12e4f1a6fe204e35500a17fcea8bcebda7481586"}, "source": ["# Data preparation\n", "To start, we read provided data. \n", "\n", "The *../input/train/* dir contains 12500 cat images and 12500 dog images.\n", "Each filename contains \"cat\" or \"dog\" as label."]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "e4d548d3-8ff4-4130-9f3e-b203aaaf79fb", "_uuid": "95dbbbe4ea8ea240c25e6bf5bb20075f20bd98ec"}, "source": ["## tool functions\n", "def ex_time(func):\n", "    start_time = datetime.datetime.now()\n", "    \n", "    def wrapper(*args, **kwargs):\n", "        print(\"start time: {}\".format(start_time))\n", "        res = func(*args, **kwargs)\n", "        \n", "        end_time = datetime.datetime.now()\n", "        ex_time = end_time - start_time\n", "        print(\"end time: {}\".format(end_time))\n", "        print(\"excute time: {} seconds\".format(ex_time.seconds))\n", "\n", "        return res\n", "       \n", "    return wrapper\n", "\n", "def display(image, image_width=IMAGE_HEIGHT, image_height=IMAGE_HEIGHT, interpolation=3):\n", "    # (784) => (28,28)\n", "    one_image = image.reshape(image_width,image_height, interpolation)\n", "    \n", "    new_f = plt.figure()\n", "    plt.axis('off')\n", "    plt.imshow(one_image, cmap=cm.binary)\n", "    plt.show()\n", "    plt.close()"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "0ca77d68-3d29-4a73-810d-83e6f0dae3e1", "_uuid": "357a381fb5f3d585cfdf50a5cf932365841cb2b9"}, "source": ["## data utility functions\n", "def dense_to_one_hot(labels_dense, num_classes):\n", "    \"\"\"\n", "    # convert class labels from scalars to one-hot vectors\n", "    # 0 => [1 0 0 0 0 0 0 0 0 0]\n", "    # 1 => [0 1 0 0 0 0 0 0 0 0]\n", "    # ...\n", "    # 9 => [0 0 0 0 0 0 0 0 0 1]\n", "    \"\"\"\n", "    num_labels = labels_dense.shape[0]\n", "    #print(\"num_labels:\", num_labels)\n", "    index_offset = np.arange(num_labels) * num_classes\n", "    #print(\"index_offset:\", index_offset)\n", "    labels_one_hot = np.zeros((num_labels, num_classes))\n", "    #print(\"labels_one_hot:\", labels_one_hot)\n", "    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n", "    #print(index_offset + labels_dense.ravel())\n", "    #print(\"labels_one_hot2:\", labels_one_hot)\n", "    return labels_one_hot\n", "\n", "def split_data(images, labels, dev_ratio=DEV_RATIO):\n", "    dev_count = int(labels.shape[1] * DEV_RATIO)\n", "    dev_images = images[:, :dev_count]\n", "    train_images = images[:, dev_count:]\n", "    dev_labels = labels[:, :dev_count]\n", "    train_labels = labels[:, dev_count:]\n", "    print(\"train images shape: {}, train labels shape:{}, \\\n", "    dev images shape: {}, dev labels shape: {}\".format(train_images.shape, train_labels.shape, dev_images.shape, dev_labels.shape))\n", "    return train_images, train_labels, dev_images, dev_labels"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "5e3c391d-730d-414a-b04f-69905466325f", "_uuid": "353e7de945ef05d124dc8830602d7ed52fec80be"}, "source": ["#@ex_time\n", "def pre_data(dirname=TRAIN_DIR, file_count=1000):\n", "    all_filenames = os.listdir(dirname)\n", "    random.shuffle(all_filenames)\n", "    filenames = all_filenames[:file_count]\n", "    \n", "    ## images\n", "    images = np.zeros((file_count, IMAGE_HEIGHT*IMAGE_WIDTH*3))\n", "    for i in range(file_count):\n", "        imgnd_origin = cv2.imread(dirname+filenames[i])\n", "        imgnd_resized = cv2.resize(imgnd_origin, (IMAGE_HEIGHT, IMAGE_WIDTH), interpolation=cv2.INTER_CUBIC)\n", "        imgnd_flatten = imgnd_resized.reshape(1,-1)\n", "        images[i] = imgnd_flatten\n", "    \n", "    ## labels from filenames\n", "    labels_list = [\"dog\" in filename for filename in filenames]\n", "    labels = np.array(labels_list, dtype='int8').reshape(file_count, 1)\n", "    \n", "    ## shuffle\n", "    permutation = list(np.random.permutation(labels.shape[0]))\n", "    shuffled_labels = labels[permutation, :]\n", "    shuffled_images = images[permutation, :]\n", "    \n", "    ## dense to one hot\n", "    labels = dense_to_one_hot(shuffled_labels, OUTPUT_SIZE)\n", "    ## normalization\n", "    images = shuffled_images/255.0\n", "    \n", "    return images.T, labels.T\n"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "2601c46a-c205-405a-a3d6-a86d6b501a4e", "_uuid": "af02ed644fa946ec111a40ba2020637e70b8a198"}, "source": ["images, labels = pre_data(file_count=100)"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "32a6e22f-4f4e-419c-a92a-b8eb2cfbae59", "_uuid": "c4980cccb3868737224d51f0d443d3f5b7b11df0"}, "source": ["train_images, train_labels, dev_images, dev_labels = split_data(images, labels)"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "25daee12-4365-4151-8bf7-8781d7ef4cf3", "_uuid": "9cae80daac699c2636f82ee3469ab994bfe6a8d3"}, "source": ["print(train_images.shape, train_labels.shape, dev_images.shape, dev_labels.shape)"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "3dc93621-f0e9-4c92-88fc-0d27c999f28a", "_uuid": "e0336ab21e49b2029891c6afb314c8559ab18291"}, "source": ["# Analysis"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "8278e07f-5054-421d-8546-d4b77d207b70", "_uuid": "e8cc83c26f1684ae083d9df973e3ea060b59872e"}, "source": ["<img src=\"../output/mems_cpu01.jpeg\" style=\"width: 100%\">"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "a4710c56-6131-4736-9bff-64ca299a0fe2", "_uuid": "4711e3e6d9b9ff4bc4a7df9459f49c6d42026838"}, "source": ["* too much memory\n", "\n", "* too much cpu"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "217350ca-8a09-42e9-a982-072f6f1eec95", "_uuid": "02d6eb12c944d4c5a37ceec3f7d07efdf9bfc242"}, "source": ["# Next"]}, {"cell_type": "markdown", "metadata": {"_cell_guid": "952e9fce-8fe2-416c-a50e-dc19eb1c29d3", "_uuid": "0678b1e151f3ce7f9d4022ee492fc7b3ed2e243d"}, "source": ["[GraphLab Create]()\n", "\n", "    * SFrame: It is an efficient disk-based tabular data structure which is not limited by RAM. It helps to scale analysis and data processing to handle large data set (Tera byte), even on your laptop.\n", "    \n", "    * SGraph: Graph helps us to understand networks by analyzing relationships between pair of items. Each item is represented by a vertex in the graph"]}, {"execution_count": null, "outputs": [], "cell_type": "code", "metadata": {"collapsed": true, "_cell_guid": "aeefd4d5-0ec7-494b-bbe6-4162742ede9c", "_uuid": "bce388948c00db6a965ccac9a16c2e6d44823fc1"}, "source": []}], "metadata": {"language_info": {"mimetype": "text/x-python", "version": "3.6.1", "name": "python", "pygments_lexer": "ipython3", "file_extension": ".py", "nbconvert_exporter": "python", "codemirror_mode": {"version": 3, "name": "ipython"}}, "kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}}}