{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "539c7005-0949-cb6c-622c-49916882da18" }, "source": [ "Use Fully-Connected Neural Network with Tensorflow" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5102b51e-ff02-c9bc-c02e-24431b1c8bf2" }, "outputs": [], "source": [ "# import dependencies\n", "import pandas as pd\n", "import numpy as np\n", "import matplotlib.pyplot as plt\n", "import pickle\n", "import tensorflow as tf\n", "from os.path import isfile, isdir\n", "% matplotlib inline" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "02b8dbeb-3ab5-d532-6e6f-323279a6a4dc" }, "outputs": [], "source": [ "# load in train & test csv\n", "train = pd.read_csv('../input/train.csv')\n", "test = pd.read_csv('../input/test.csv')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "37d0a926-6391-d9cf-211f-1bccaf9212d3" }, "outputs": [], "source": [ "# check training data\n", "trainLabelCounts = train['label'].value_counts(sort = False)\n", "trainLabelCounts" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "27f0b627-fe62-5969-302e-0b6fd9739ffd" }, "outputs": [], "source": [ "# check training data - plot\n", "def getImage(data, *args):\n", " '''\n", " Get the image by specified number (Randomly)\n", " parameters:\n", " data: dataframe\n", " number: int, the label of the number to show\n", " output: 1-D numpy array\n", " '''\n", " if args:\n", " number = args[0]\n", " specifiedData = data[data['label'] == number].values\n", " else:\n", " specifiedData = data.values\n", " randomNumber = np.random.choice(len(specifiedData)-1, 1)\n", " return specifiedData[randomNumber,:]\n", " \n", "def plotNumber(imageData, imageSize):\n", " '''\n", " parameters:\n", " data: label & 1-D array of pixels\n", " '''\n", " # show the image of the data\n", " if imageData.shape[1] == np.prod(imageSize):\n", " image = imageData[0,:].reshape(imageSize)\n", " elif imageData.shape[1] > np.prod(imageSize):\n", " label = imageData[0,0]\n", " image = imageData[0,1:].reshape(imageSize)\n", " plt.title('number: {}'.format(label))\n", " plt.imshow(image)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "cf7a5356-3361-f3b0-7330-7ddf464de9b0" }, "outputs": [], "source": [ "# plot the training image with specified number\n", "imageSize = (28, 28)\n", "chosenNumber = 1\n", "plotNumber(getImage(train, chosenNumber), imageSize)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "3f5552c1-e18f-7daf-e022-4641b5ac7a41" }, "outputs": [], "source": [ "# check testing data\n", "plotNumber(getImage(test), imageSize)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "521416fe-85f8-b5d9-e6d6-b98b944e6f74" }, "source": [ "Preprocessing" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ffbc3486-6ad5-b015-ed98-b09e88361991" }, "outputs": [], "source": [ "# cast to numpy array\n", "trainData = train.values[:,1:]\n", "trainLabel = train.values[:,0]\n", "testData = test.values" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f1b27e4c-043f-218d-5278-57040d38f0d6" }, "outputs": [], "source": [ "def preprocessing(data):\n", " '''\n", " min-max scaling for every image\n", " data: numpy array\n", " output: scaled numpy array\n", " '''\n", " minV = 0\n", " maxV = 255\n", " data = (data - minV) / (maxV-minV)\n", " return data\n", "def one_hot_encoding(data, numberOfClass):\n", " from sklearn import preprocessing\n", " lb = preprocessing.LabelBinarizer()\n", " lb.fit(range(numberOfClass))\n", " return lb.transform(data)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c18042d2-3662-65bb-0d84-fe292385fdb4" }, "outputs": [], "source": [ "processedTrainData = preprocessing(trainData)\n", "processedTestData = preprocessing(testData)\n", "one_hot_trainLabel = one_hot_encoding(trainLabel, 10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "58685c92-2d08-5feb-3f4f-09d7aa2d1eb9" }, "outputs": [], "source": [ "# save in pickle\n", "fileName = 'mnist.p'\n", "if not isfile(fileName):\n", " pickle.dump((processedTrainData, trainLabel, one_hot_trainLabel, processedTestData), open(fileName, 'wb'))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "adc630dd-b9d1-114e-ca7f-ca85c5f69c80" }, "source": [ "Checkpoint" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "b50f514d-7972-3e41-4405-5c3207e128f9" }, "outputs": [], "source": [ "# load pickle file\n", "fileName = 'mnist.p'\n", "trainData, trainLabel, one_hot_trainLabel, testData = pickle.load(open(fileName, mode = 'rb'))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "db4e28ab-10ae-33a9-2c35-cc34718b2ada" }, "source": [ "Build Neural Network with Tensorflow" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0c515a91-0cd5-a717-bbf2-ac664efa3fcb" }, "outputs": [], "source": [ "def getInputTensor(features, numberOfClass):\n", " '''\n", " Create tf.placeholder for input & label\n", " '''\n", " inputT = tf.placeholder(dtype = tf.float32, shape = (None, features), name = 'input')\n", " labelT = tf.placeholder(dtype = tf.float32, shape = (None, numberOfClass), name = 'label')\n", " keep_prob = tf.placeholder(dtype = tf.float32)\n", " \n", " return inputT, labelT, keep_prob" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "54ebd334-0b86-dfdc-a210-8efb9535081d" }, "outputs": [], "source": [ "def hiddenLayer(inputT, numberOfNodes):\n", " '''\n", " Create hidden layer\n", " '''\n", " inputSize = inputT.get_shape().as_list()[1]\n", " # create weights & biases\n", " weights = tf.Variable(tf.truncated_normal((inputSize, numberOfNodes)), dtype = tf.float32)\n", " biases = tf.zeros((numberOfNodes), dtype = tf.float32)\n", " # output of hidden nodes\n", " hiddenNodes = tf.add(tf.matmul(inputT, weights), biases)\n", " hiddenOutput = tf.nn.sigmoid(hiddenNodes)\n", " \n", " return hiddenOutput" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ecf76352-3136-c9b2-7ecf-c754d2ab45eb" }, "outputs": [], "source": [ "def outputLayer(hiddenOutput, numberOfClass):\n", " '''\n", " Create output layer\n", " hiddenOutput: output from hidden layer\n", " numOfClass: number of classes (0~9)\n", " '''\n", " inputSize = hiddenOutput.get_shape().as_list()[1]\n", " # create weights & biases\n", " weights = tf.Variable(tf.truncated_normal((inputSize, numberOfClass)), dtype = tf.float32)\n", " biases = tf.zeros((numberOfClass), dtype = tf.float32)\n", " # output\n", " output = tf.add(tf.matmul(hiddenOutput, weights), biases)\n", " \n", " return output" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9013b9e3-b74a-bf0c-c6d3-311aeeee7d85" }, "outputs": [], "source": [ "def build_nn(inputT, numberOfNodes, numberOfClass, keep_prob):\n", " '''\n", " build fully connected neural network\n", " '''\n", " # fully_connected layers\n", " fc1 = hiddenLayer(inputT, numberOfNodes)\n", " output = outputLayer(fc1, numberOfClass)\n", " \n", " return output" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "024be97d-842e-af40-6f57-0f3bed701374" }, "source": [ "Hyperparameter" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8a0dac83-7362-1b86-8ff2-2435978ff6c4" }, "outputs": [], "source": [ "numberOfNodes = 256\n", "batchSize = 128\n", "numberOfEpoch = 20\n", "learningRate = 0.01\n", "keep_prob_rate = 1.0" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ad0e67f1-ae63-2e37-b445-b6ce00cfebe4" }, "outputs": [], "source": [ "# Build Neural Network graph\n", "numberOfClass = 10\n", "imageSize = (28, 28)\n", "features = np.prod(imageSize)\n", "graph = tf.Graph()\n", "tf.reset_default_graph()\n", "with graph.as_default():\n", " \n", " # get inputs\n", " inputT, labelT, keep_prob = getInputTensor(features, numberOfClass)\n", " \n", " # build fully-conneted neural network\n", " logits = build_nn(inputT, numberOfNodes, numberOfClass, keep_prob)\n", " \n", " # softmax with cross entropy\n", " probability = tf.nn.softmax(logits, name = 'probability')\n", " \n", " # Cost\n", " cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = labelT))\n", " \n", " # optimizer\n", " optimizer = tf.train.AdamOptimizer(learning_rate = learningRate).minimize(cost)\n", " \n", " # accuracy\n", " correctPrediction = tf.equal(tf.argmax(probability, 1),tf.argmax(labelT, 1))\n", " accuracy = tf.reduce_mean(tf.cast(correctPrediction, tf.float32))" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "fdd10084-f0c9-3c11-9066-199acaee8879" }, "source": [ "Train" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "05bca27c-ef63-5ac7-e4bb-d39d1cf6c5f9" }, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c1e4ba33-1b1c-6534-adbd-43979df15fc8" }, "outputs": [], "source": [ "def printResult(epoch, numberOfEpoch, trainLoss, validationLoss, validationAccuracy):\n", " print(\"Epoch: {}/{}\".format(epoch+1, numberOfEpoch),\n", " '\\tTraining Loss: {:.3f}'.format(trainLoss),\n", " '\\tValidation Loss: {:.3f}'.format(validationLoss),\n", " '\\tAccuracy: {:.2f}%'.format(validationAccuracy*100))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "814de6b5-b87e-90b2-4f32-485d47d09de6" }, "outputs": [], "source": [ "save_dir = './save'\n", "with tf.Session(graph=graph) as sess:\n", " sess.run(tf.global_variables_initializer())\n", " \n", " for epoch in range(numberOfEpoch):\n", " # training data & validation data\n", " train_x, val_x, train_y, val_y = train_test_split(trainData, one_hot_trainLabel,\\\n", " test_size = 0.2) \n", " # training loss\n", " for i in range(0, len(train_x), batchSize):\n", " trainLoss, _, _ = sess.run([cost, probability, optimizer], feed_dict = {\n", " inputT: train_x[i: i+batchSize],\n", " labelT: train_y[i: i+batchSize],\n", " keep_prob: keep_prob_rate \n", " })\n", " \n", " # validation loss\n", " valAcc, valLoss = sess.run([accuracy, cost], feed_dict ={\n", " inputT: val_x,\n", " labelT: val_y,\n", " keep_prob: 1.0\n", " })\n", " \n", " \n", " # print out\n", " printResult(epoch, numberOfEpoch, trainLoss, valLoss, valAcc)\n", " # save\n", " saver = tf.train.Saver()\n", " saver.save(sess, save_dir)" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "9210eeb6-40a1-b7f2-6726-b643401da0e8" }, "source": [ "## Test" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "176421ab-2b55-113c-6247-2f8c6094c17d" }, "outputs": [], "source": [ "# test result\n", "\n", "loaded_Graph = tf.Graph()\n", "with tf.Session(graph=loaded_Graph) as sess:\n", " loader = tf.train.import_meta_graph(save_dir +'.meta')\n", " loader.restore(sess, save_dir) \n", " # get tensors\n", " loaded_x = loaded_Graph.get_tensor_by_name('input:0')\n", " loaded_y = loaded_Graph.get_tensor_by_name('label:0')\n", " loaded_prob = loaded_Graph.get_tensor_by_name('probability:0')\n", " \n", " prob = sess.run(tf.argmax(loaded_prob,1), feed_dict = {loaded_x: testData})" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "0dc70545-337c-aaec-1fb7-13bac1c18deb" }, "outputs": [], "source": [ "which = 1\n", "print('predicted labe: {}'.format(str(prob[which])))\n", "plotNumber(np.array([testData[which, :]]),imageSize)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "5ed49908-63e1-4210-01d5-2a7c2d20eaa5" }, "outputs": [], "source": [ "print(prob)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "9c24f556-4956-5954-443c-f1c533a58d40" }, "outputs": [], "source": [ "import csv\n", "outputFile_dir = 'output.csv'\n", "header = ['ImageID','Label']" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "12dabc9b-1eac-72c5-0a3c-cdd8c8508f63" }, "outputs": [], "source": [ "with open(outputFile_dir, 'w') as csvFile:\n", " writer = csv.writer(csvFile, delimiter = ',')\n", " writer.writerow(header)\n", " for i, p in enumerate(prob):\n", " writer.writerow([str(i+1), str(p)])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "8d708d52-a768-2140-4b6a-f42adf612160" }, "outputs": [], "source": [ "" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }