{"nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}, "language_info": {"mimetype": "text/x-python", "file_extension": ".py", "name": "python", "version": "3.6.2rc2", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "codemirror_mode": {"name": "ipython", "version": 3}}}, "nbformat_minor": 1, "cells": [{"outputs": [], "source": ["import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import tensorflow as tf #Conv Net\n", "import tflearn\n", "import os\n", "import pickle\n", "from sklearn.model_selection import train_test_split\n", "from skimage import color\n", "from skimage.transform import resize\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "1a4c5c1d-c2e6-4a30-b7fd-024fc316dc83", "_uuid": "d5db9cc37ebbf64e63cbc3838618254f51e91734"}}, {"cell_type": "markdown", "source": ["All dependencies needed for this kernel"], "metadata": {"_cell_guid": "82122ab1-bd40-4aaf-b309-8a3c7ca056fb", "_uuid": "72cc27d7f700fda096c24a07a081f2a1c15cbf37"}}, {"outputs": [], "source": ["from subprocess import check_output\n", "path = \"D:/Ramses/Documents/GPU_TensorFlow/Kaggle/DogBreedClassifier\"\n", "os.path.exists(path)"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "42a7c305-8405-491a-bd22-95d39956f94d", "_uuid": "6251a25e6103a20b22d23039215ea333c38028b1"}}, {"cell_type": "markdown", "source": ["1. DATA PREPROCESSING:"], "metadata": {"_cell_guid": "43bf7e32-7050-4e2d-8977-f8453304b369", "_uuid": "52968e73de9995316d9a0921634be4e1c0e4dc5a"}}, {"outputs": [], "source": ["#CONSTANTS\n", "image_shape = (100, 100)\n", "channels = 3\n", "alpha = 0.01\n", "sample_csv = None\n", "df_train = None\n", "df_test = None\n", "df_train_labels = None"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "4d066a60-9577-4f4d-af10-91550a77cea5", "_uuid": "1681e882497e47c1716d020ec48cf31a29c10152"}}, {"cell_type": "markdown", "source": ["Just to have an idea of what the submission file looks like\n", "\n", "+Reading in the training data's labels\n", "\n", "Input data files are available in the \"../input/\" directory."], "metadata": {"_cell_guid": "e8a1d224-1321-4cc1-93d8-d19217902c44", "_uuid": "e84d9ae4e79bbba62f3af5cab60a059a974c6d7a"}}, {"outputs": [], "source": ["def get_sample_csv():\n", "    sample = pd.read_csv(path + '/sample_submission.csv')\n", "    return sample\n", "def get_training_data():\n", "    labels = pd.read_csv(path + '/labels.csv', index_col=0)\n", "    return labels\n", "\n", "df_train_labels = get_training_data()\n", "sample_csv = get_sample_csv()"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "31dd0259-5e83-4bbb-80a0-66908210e05b", "_uuid": "87bec086bdac9e10bfc90499098caa9c7942b8c5"}}, {"cell_type": "markdown", "source": ["tqdm is for the nice-looking progress bar\n", "\n", "Data preprocessing and extraction:\n", "    * Get the image and its id\n", "    * Resize the image to be 100 x 100 x 3\n", "    * Store the image in pandas.DataFrame with id as index and 'image' as column\n", "    * Store the ids in a separate numpy array to be reused later"], "metadata": {"_cell_guid": "942705eb-b956-4804-8429-a4b9b28ba2ae", "_uuid": "9153461d271c06f7ce1a75f4485faf71fa1ce0ac"}}, {"outputs": [], "source": ["from tqdm import tqdm\n", "#Getting the train and test data\n", "def image_preprocessing(image):\n", "    #image = color.rgb2gray(image)\n", "    image = resize(image, image_shape, mode='constant')\n", "    return image\n", "    \n", "def get_train_test_df(): #Maybe store in a pickle?\n", "    train, test = pd.DataFrame(dtype=object,columns=['image']), pd.DataFrame(dtype=object, columns=['image'])\n", "    train_ids, test_ids = np.array([], dtype=object), np.array([], dtype=object)\n", "    for i, p in enumerate([path + '/train', path + '/test']):\n", "        for f in tqdm(os.listdir(p)):\n", "            _id = os.path.split(f)[-1]\n", "            _id = str(_id.split('.', 1)[0])\n", "            entry = plt.imread(p + '/' + f, format='jpg')\n", "            entry = image_preprocessing(entry)\n", "            if i is 0: \n", "                train = train.append({'image': entry}, ignore_index=True)\n", "                train_ids = np.append(train_ids, _id)\n", "            else: \n", "                test = test.append({'image': entry}, ignore_index=True)\n", "                test_ids = np.append(test_ids, _id)\n", "    return train, test, train_ids, test_ids\n", "\n", "df_train, df_test, df_train_ids, df_test_ids = get_train_test_df()\n", "print('Done.')"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "855226bd-3940-497a-9482-26f9c8a25f2d", "_uuid": "8f4cb9b46aee2273e5b265831cae241df8a31c2b"}}, {"cell_type": "markdown", "source": ["Save the data to separate pickle files so that the extraction doesn't have to be done everytime:"], "metadata": {"_cell_guid": "6b2c7560-9a40-48df-99d2-8e12c525be97", "_uuid": "74426e0b3c08dff9817e92d9fd835fd6f40296ef"}}, {"outputs": [], "source": ["for name in ['df_train', 'df_test', 'df_train_ids', 'df_test_ids']: \n", "    with open(path + '/' + name + '.pickle', 'wb') as f:\n", "        pickle.dump(globals()[name], f)"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "ce00253a-bc8f-4e07-96dd-52305e6f1f81", "_uuid": "2662ad63fc23cbe3978c0ca2f6670c255b183829"}}, {"cell_type": "markdown", "source": ["Load the data if needed:"], "metadata": {"_cell_guid": "afdd116b-e585-4b2f-9f14-3be4f5b5934a", "_uuid": "3e50f09d39d1d862a53dea5b034c93281718027e"}}, {"outputs": [], "source": ["if os.path.exists(path + '/df_train.pickle'):\n", "    for name in ['df_train', 'df_test', 'df_train_ids', 'df_test_ids']: \n", "        with open(path + '/' + name + '.pickle', 'rb') as f:\n", "            globals()[name] = pickle.load(f)"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "3ad4731e-7e78-4ae8-b1c9-91a0c3e9636a", "_uuid": "261adac4b718bac136e8a244478c821d940cebcb"}}, {"cell_type": "markdown", "source": ["A quick check that the data looks the way we want it to:"], "metadata": {"_cell_guid": "2e613dca-d3d1-4044-9d80-c80e121d5d34", "_uuid": "12b40efc04f7c6807553a7251473027fb2932347"}}, {"outputs": [], "source": ["img = df_train.iloc[0]['image']\n", "plt.imshow(img)"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "164a67df-f63f-4a19-90cd-5d4eb4118b61", "_uuid": "e9941f92fe356b9d5ddf5fcf1504be96f3fda685"}}, {"cell_type": "markdown", "source": ["Getting all the labels:\n", "    * Realizations:\n", "        * Data set contains too many labels with too little data to train a convnet on\n", "        * Other ML techniques would be possible at this stage\n", "        * Gathering more data would be a possibility\n", "        * Reducing the number of targets to a decent amount is possible\n", "    * Solution I chose:\n", "        * Divided the data into 12 targets each containing 10 sub-targets\n", "        * Division is done alphabetically (There are better ways but they require a deeper insight on the data)\n", "        * Network is only going to predict one of these 12 general-classes\n", "        * If I have time, I'll improve it to use a K-Nearest Neighbour algorithm\n", "          to determine which of the 10 subclasses the input belongs to (or maybe an array of dedicated convnets)"], "metadata": {"_cell_guid": "635be28f-b236-45db-ad58-2449b2a63f43", "_uuid": "a2b3d7c6cc3200237688425657a64afa726d8a89"}}, {"outputs": [], "source": ["df_labels = []\n", "for row in df_train_labels.itertuples():\n", "    if row[-1] not in df_labels:\n", "        df_labels.append(row[-1])\n", "df_labels = np.asarray(df_labels)\n", "df_labels = np.sort(df_labels)\n", "df_labels = np.split(df_labels, 12)"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "829e6a9f-98a7-4d8e-9536-84932d0da06f", "_uuid": "500b4b03d358af7abf4a0ef318e14bafa0f7dd54"}}, {"cell_type": "markdown", "source": ["Quick check to see how many samples the biggest general-class contains"], "metadata": {"_cell_guid": "fed0272a-59d6-448a-85da-bfd764809c86", "_uuid": "6db4000acace92e7d8ca86e141c2f782df08e870"}}, {"outputs": [], "source": ["from collections import Counter\n", "occurances = np.zeros(12)\n", "values = []\n", "for value in df_train_labels.values:\n", "    values.append(value[0])\n", "for index, label in enumerate(df_labels):\n", "    for value in values:\n", "        if value in label:\n", "            occurances[index] += 1\n", "print(max(occurances))"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "9a7913c4-1b46-4a33-b899-502a77f1df4e", "_uuid": "8055fc08ee457659c95e0d578e2492fea97b4c3c"}}, {"cell_type": "markdown", "source": ["Train-test-validation split of the data (I used 80-20 because we already don't have a lot of training data for each class)"], "metadata": {"_cell_guid": "c43d2c38-f543-485b-aa8e-0d31c20a6dc1", "_uuid": "dc0fdafaa9cfa918b3527c91375c22706ec598b2"}}, {"outputs": [], "source": ["def get_train_test_split(test_size=0.5):\n", "    index_split = int(df_train.size * (1 - test_size))\n", "    train_input, test_input = np.split(df_train.values, [index_split])\n", "    train_targets, test_targets = np.split(df_train_labels.values, [index_split])\n", "    assert len(test_input) == len(test_targets)\n", "    return train_input, test_input, train_targets, test_targets\n", "\n", "network_input, validation_input, network_targets, validation_targets = get_train_test_split(test_size=0.2)"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "738b3f6d-9437-4d6e-9741-52dc7ae8a7fe", "_uuid": "875fa8838666a3dce4af1a09c440f82b66f3da90"}}, {"cell_type": "markdown", "source": ["Converting the targets to be one-hot arrays to facilitate the prediction:"], "metadata": {"_cell_guid": "30795973-c8de-41f8-b0cf-0abd0536cb25", "_uuid": "be43001ffacf1aa84f784518098e4957bf45540b"}}, {"outputs": [], "source": ["#make targets a one-hot array\n", "def get_one_hot_array(array):\n", "    one_hot = []\n", "    for entry in array:\n", "        for i, group in enumerate(df_labels):\n", "            if entry[0] in group:\n", "                index = i\n", "                break\n", "        zero = np.zeros(len(df_labels))\n", "        zero[index] = 1\n", "        one_hot.append(zero)\n", "    return one_hot\n", "network_targets = get_one_hot_array(network_targets)\n", "validation_targets = get_one_hot_array(validation_targets)\n", "network_targets[0]"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "0c5834f4-1787-4b13-bc7e-59b58602094f", "_uuid": "cb50253fa65e2f42bb4bcf620f475401954debee"}}, {"cell_type": "markdown", "source": ["Add dimension to network_input so it becomes a 4D array and is compatible with the convnet:"], "metadata": {"_cell_guid": "c13cf692-edbb-4c6f-ab66-f5cf6f4135cc", "_uuid": "c9e161f76ea49a0b736d2496f8e14e6a9639b2f4"}}, {"outputs": [], "source": ["def get_correct_dimensions(array):\n", "    inputs = []\n", "    for arr in array:\n", "        inputs.append(arr[0])\n", "    inputs = np.array(inputs)\n", "    return inputs\n", "\n", "network_input = get_correct_dimensions(network_input)\n", "validation_input = get_correct_dimensions(validation_input)\n", "print(network_input.shape)\n", "print(validation_input.shape)"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "e8bbe3a3-93d8-4cac-aaa4-949614433940", "_uuid": "c7bdfd2dd59221d492e0b0860e7797b4de8f5e0d"}}, {"cell_type": "markdown", "source": ["2 BUILDING THE NETWORK AND TRAINING"], "metadata": {"_cell_guid": "6bc88f56-f426-42f6-8bed-2c9254159819", "_uuid": "9789cf06c71ffa91a4e7cf187535630ffd68ea04"}}, {"cell_type": "markdown", "source": ["Building the network:\n", "    * Structure:\n", "        * Simplest Convnet with dropout\n", "        * Layers:\n", "            - 4 Convolutional Layers\n", "            - 4 Fully Connected\n", "        * Optimizer:\n", "            - Adam\n", "    * Engine:\n", "        * TensorFlow (through TFLearn)\n", "        * Why?\n", "            - Simple to use\n", "            - Great for a first pass through the data\n", "            - Very intuitive to read for new comers"], "metadata": {"_cell_guid": "86e6c718-b934-4797-9a8c-c2bda3950988", "_uuid": "5e23de933e03ad28792d0cad16e918f42ed74d88"}}, {"outputs": [], "source": ["from tflearn.layers.conv import conv_2d, max_pool_2d\n", "from tflearn.layers.core import input_data, dropout, fully_connected\n", "from tflearn.layers.estimator import regression\n", "from tflearn.layers.normalization import local_response_normalization\n", "\n", "def build_conv_net(shape, channels=5, n_features=1, learning_rate=0.1):\n", "    inputs = input_data(shape=[None, shape[0], shape[1], channels], name='input')\n", "    network = conv_2d(inputs, 5, 5, activation='relu')\n", "    network = max_pool_2d(network, 3, strides=2)\n", "    network = local_response_normalization(network)\n", "    network = conv_2d(network, 5, 5, activation='relu')\n", "    network = max_pool_2d(network, 3, strides=2)\n", "    network = local_response_normalization(network)\n", "    network = conv_2d(inputs, 5, 5, activation='relu')\n", "    network = max_pool_2d(network, 3, strides=2)\n", "    network = local_response_normalization(network)\n", "    network = conv_2d(network, 5, 5, activation='relu')\n", "    network = max_pool_2d(network, 3, strides=2)\n", "    network = local_response_normalization(network)\n", "    network = fully_connected(network, 256, activation='relu')\n", "    network = dropout(network, 0.8)\n", "    network = fully_connected(network, 256, activation='relu')\n", "    network = dropout(network, 0.8)\n", "    network = fully_connected(network, 256, activation='tanh')\n", "    network = dropout(network, 0.8)\n", "    network = fully_connected(network, n_features, activation='softmax')\n", "    network = regression(network, optimizer='adam', learning_rate=learning_rate, loss='categorical_crossentropy', name='targets')\n", "    model = tflearn.DNN(network)\n", "    return model"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "caddc5f0-904c-4eb0-8cb1-f28fdbce3c4e", "_uuid": "ad2d5ffd9c8f7ee675ec5b603d5000f6da792942"}}, {"cell_type": "markdown", "source": ["Creating the model:"], "metadata": {"_cell_guid": "ea7f33b1-e7bb-499b-929a-c3277cbc950d", "_uuid": "1fe0011b4efbce61781d15b450759075fceabdd8"}}, {"outputs": [], "source": ["n_features = len(df_labels)\n", "alpha = 0.001\n", "convnet = build_conv_net(image_shape, channels=channels, n_features=n_features, learning_rate=alpha)"], "cell_type": "code", "execution_count": null, "metadata": {"scrolled": true, "_cell_guid": "875865e0-1b5c-4b83-b090-80d407d97cd0", "_uuid": "2f0acc5c1ef3f73c17558d6aab26a4079e133afe"}}, {"cell_type": "markdown", "source": ["Training of the model:"], "metadata": {"_cell_guid": "7ee3f333-d756-49e9-9b36-3c592f42a673", "_uuid": "905d2838bac8ddf0ab7a957150c739c35811f994"}}, {"outputs": [], "source": ["def train_network(model):\n", "    model.fit({'input': network_input},\n", "              {'targets': network_targets},\n", "              n_epoch=1,\n", "              batch_size=32,\n", "              validation_set=({'input': validation_input}, {'targets': validation_targets}),\n", "              snapshot_step=500,\n", "              show_metric=True,\n", "              run_id='dog_breed_classifier')\n", "    model.save('dogs.model')"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "d958179e-7b14-486d-a287-92f094329bf4", "_uuid": "c8d8b566a873b8914da4eaeab0eec5178973035b"}}, {"outputs": [], "source": ["train_network(convnet)"], "cell_type": "code", "execution_count": null, "metadata": {"_cell_guid": "00896e2d-7e50-40ae-8dfd-4cb85aa3b639", "_uuid": "6193eb5795e9b1322b17fdea3a16cc8f16f9b08a"}}, {"cell_type": "markdown", "source": ["We can see the model achieves terrible accuracy but:\n", "    * Training was only for 1 epoch\n", "    * Wasn't designed to achieve 100% accuracy\n", "    * Data set is fairly small for each target\n", "So results are within expectations"], "metadata": {"_cell_guid": "074e9f0f-98e7-4a31-9491-bf0228dccd3a", "_uuid": "b78c44d4121bbdafb63691ba410c0755068d2740"}}, {"cell_type": "markdown", "source": ["3 TESTING THE MODEL AND CREATING SUBMISSION FILE"], "metadata": {"_cell_guid": "a923c9c5-ae3a-46cc-91fb-464df13aefb5", "_uuid": "c456d40dcd0280bba97d8b289f0fdf8bd0bbf5d0"}}, {"cell_type": "markdown", "source": ["Collecting the predictions for the testing data:"], "metadata": {"_cell_guid": "c160ef0b-e41b-41be-89ff-3389f554f2e7", "_uuid": "a157d0b859949cadae0bf5df5b722de214cb2eba"}}, {"outputs": [], "source": ["def test_model(model):\n", "    try: model.load('dogs.model')\n", "    except: print('Model not found.')\n", "    predictions = model.predict(df_test)\n", "    return predictions"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "e2486afd-14c9-4db3-ba67-e495f1a70ada", "_uuid": "bdb3888aa8580dde534ecfe21c2be1bef28a4384"}}, {"outputs": [], "source": ["test_results = test_model(convnet)"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "e9438c9a-cac8-4034-b42b-7fb24f5c33f5", "_uuid": "ee6402dbbe4740348f2fcf5b967a53a5215e8d4a"}}, {"cell_type": "markdown", "source": ["Making the submission file:\n", "    * Store the results in a pandas.DataFrame with index the id and columns the targets\n", "    * Store the results in a csv using pandas.to_csv()"], "metadata": {"_cell_guid": "1a7bb890-f9e5-4006-b61d-ce0b0d592b1a", "_uuid": "23c67ea8876f6929e4dad698620692c256105077"}}, {"outputs": [], "source": ["def make_submission_file():\n", "    df_predictions = pd.DataFrame(index=df_test.values, columns=df_labels)\n", "    for i, _id in enumerate(df_test_ids):\n", "        for j, label in enumerate(df_labels):\n", "            df_predictions.at[label, _id] = test_results[i][j]\n", "    df_predictions.to_csv(path_or_buf='submission_file.csv')\n", "make_submission_file()"], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "ab780a6e-c0d1-4bc1-91bb-877ad4dc497e", "_uuid": "d25b77421826ad0f4388669b9011aed623b657b2"}}, {"outputs": [], "source": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "a40e8230-6d3d-41a4-9516-f0ef96c90f61", "_uuid": "dea1d227807291e94ed8c1ff3d8945108da0abff"}}, {"outputs": [], "source": [], "cell_type": "code", "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "a2a10152-483c-45af-8ec3-19ca2bfff5e3", "_uuid": "dd32a9d66ce1ff5c3366054c5a2795c9a2e5cfb7"}}]}