{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"collapsed":true},"cell_type":"code","source":"# import dependencies\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pickle\nimport tensorflow as tf\nfrom os.path import isfile, isdir\n% matplotlib inline","execution_count":7,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"# load in train & test csv\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":8,"outputs":[]},{"metadata":{"_cell_guid":"02677c80-53bb-44ed-a586-106b694ef40f","_uuid":"5bfb95a85de48bf058231d01c9c1d9a28c8430c5","trusted":true},"cell_type":"code","source":"# check training data\ntrainingLabelCounts = train['label'].value_counts(sort = False)\ntrainingLabelCounts","execution_count":9,"outputs":[]},{"metadata":{"_cell_guid":"0596b166-e88e-4947-8135-4ad518b8153e","_uuid":"32b55e81ae3949a8646d3dfa705aaf63564bf5a9","collapsed":true,"trusted":true},"cell_type":"code","source":"# check training data - plot\ndef getImage(data, *args):\n    '''\n    Get the image by specified number (Randomly)\n    parameters:\n        data: dataframe\n        number: int, the label of the number to show\n    output: 1-D numpy array\n    '''\n    if args:\n        number = args[0]\n        specifiedData = data[data['label'] == number].values\n    else:\n        specifiedData = data.values\n    randomNumber = np.random.choice(len(specifiedData)-1, 1)\n    return specifiedData[randomNumber,:]\n    \ndef plotNumber(imageData, imageSize):\n    '''\n    parameters:\n        data: label & 1-D array of pixels\n    '''\n    # show the image of the data\n    if imageData.shape[1] == np.prod(imageSize):\n        image = imageData[0,:].reshape(imageSize)\n    elif imageData.shape[1] > np.prod(imageSize):\n        label = imageData[0,0]\n        image = imageData[0,1:].reshape(imageSize)\n        plt.title('number: {}'.format(label))\n    plt.imshow(image)","execution_count":10,"outputs":[]},{"metadata":{"_cell_guid":"c5236f42-5bb7-4b13-b8d4-50713e9217d0","_uuid":"b9affa3ef3b44112b811b5c25dbfcba1f8bedbb5","collapsed":true,"trusted":true},"cell_type":"code","source":"trainData = train.values[:,1:]\ntrainLabel = train.values[:,0]\ntestData = test.values\n\ndef preprocessing(data):\n    minV = 0\n    maxV = 255\n    data = (data - minV) / (maxV - minV)\n    return data\n\ndef one_hot_encoding(data, numberOfClass):\n    from sklearn import preprocessing\n    lb = preprocessing.LabelBinarizer()\n    lb.fit(range(numberOfClass))\n    return lb.transform(data)\n\nprocessedTrainData = preprocessing(trainData)\nprocessedTestData = preprocessing(testData)\none_hot_trainLabel = one_hot_encoding(trainLabel, 10)\n\n# save in pickle\n\nfileName = \"mnist.p\"\nif not isfile(fileName):\n    pickle.dump((processedTrainData, trainLabel, one_hot_trainLabel, processedTestData), open(fileName, \"wb\"))","execution_count":11,"outputs":[]},{"metadata":{"_cell_guid":"3a0e23e8-29f2-4a2d-a252-781a86d9dcb1","_uuid":"a411ad655c24d11bc19270c9a5a92e4bcc74daaf","collapsed":true,"trusted":true},"cell_type":"code","source":"# load pickle file\n\nfileName = \"mnist.p\"\ntrainData, trainLabel, one_hot_trainLabel, testData = pickle.load(open(fileName, mode = \"rb\"))\n\ndef getInputTensor(features, numberOfClass):\n    inputT = tf.placeholder(dtype = tf.float32, shape = (None, features), name = \"input\")\n    labelT = tf.placeholder(dtype = tf.float32, shape = (None, numberOfClass), name = \"label\")\n    keep_prob = tf.placeholder(dtype = tf.float32)\n    \n    return inputT, labelT, keep_prob\n\ndef hiddenLayer(inputT, numberOfNodes):\n    inputSize = inputT.get_shape().as_list()[1]\n    #create weights & biases\n    weights = tf.Variable(tf.truncated_normal((inputSize, numberOfNodes)), dtype = tf.float32)\n    biases = tf.zeros((numberOfNodes), dtype = tf.float32)\n    # output of hidden nodes\n    hiddenNodes = tf.add(tf.matmul(inputT, weights), biases)\n    hiddenOutput = tf.nn.sigmoid(hiddenNodes)\n    \n    return hiddenOutput\n\ndef outputLayer(hiddenOutput, numberOfClass):\n    inputSize = hiddenOutput.get_shape().as_list()[1]\n    # create weights & biases\n    weights = tf.Variable(tf.truncated_normal((inputSize, numberOfClass)), dtype = tf.float32)\n    biases = tf.zeros((numberOfClass), dtype = tf.float32)\n    # output\n    output = tf.add(tf.matmul(hiddenOutput, weights), biases)\n    \n    return output\n\ndef build_nn(inputT, numberOfNodes, numberOfClass, keep_prob):\n    #fully_connected layers\n    \n    fc1 = hiddenLayer(inputT, numberOfNodes)\n    output = outputLayer(fc1, numberOfClass)\n    \n    return output","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"927c6637-d876-4f60-bc44-e7470831d7a0","_uuid":"02d2eb66fa2adb4195a6fc882bac4e99d75d9626","collapsed":true,"trusted":true},"cell_type":"code","source":"# HyperParameters\n\nnumberOfNodes = 256\nbatchSize = 128\nnumberOfEpoch = 20\nlearningRate = 0.01\nkeep_prob_rate = 1.0","execution_count":13,"outputs":[]},{"metadata":{"_cell_guid":"1f5420e2-8b5f-46be-bd38-46565d075915","_uuid":"d45b3330c5672273f75d74f20974eef457204772","trusted":true},"cell_type":"code","source":"# Build Neural Network graph\n\nnumberOfClass = 10\nimageSize = (28, 28)\nfeatures = np.prod(imageSize)\ngraph = tf.Graph()\ntf.reset_default_graph()\nwith graph.as_default():\n    #get inputs\n    inputT, labelT, keep_prob = getInputTensor(features, numberOfClass)\n    \n    #build fully-connected neural network\n    logits = build_nn(inputT, numberOfNodes, numberOfClass, keep_prob)\n    \n    #softmax with cross entropy\n    probability = tf.nn.softmax(logits, name = \"probability\")\n    \n    # cost\n    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = labelT))\n    \n    # optimizer\n    optimizer = tf.train.AdamOptimizer(learning_rate = learningRate).minimize(cost)\n    \n    # accuracy\n    correctPrediction = tf.equal(tf.argmax(probability, 1), tf.argmax(labelT, 1))\n    accuracy = tf.reduce_mean(tf.cast(correctPrediction, tf.float32))","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"fe540425-b548-4ec3-9a75-6819f8ad1b28","_uuid":"fe44e7d15587a25f52bd96cba1b2e3b59d485db8","trusted":true},"cell_type":"code","source":"#train\n\nfrom sklearn.model_selection import train_test_split\n\ndef printResult(epoch, numberOfEpoch, trainLoss, validationLoss, validationAccuracy):\n    print(\"Epoch: {}/{}\".format(epoch + 1, numberOfEpoch),\n         \"\\tTraining Loss: {:.3f}\".format(trainLoss),\n         \"\\tValidation Loss: {:.3f}\".format(validationLoss),\n         \"\\tAccuracy: {:.2f}%\".format(validationAccuracy * 100))\n\nsave_dir = \"./save\"\nwith tf.Session(graph=graph) as sess:\n    sess.run(tf.global_variables_initializer())\n    \n    for epoch in range(numberOfEpoch):\n        #training data & validation data\n        train_x, val_x, train_y, val_y = train_test_split(trainData, one_hot_trainLabel, test_size = 0.2)\n        \n        #training loss\n        for i in range(0, len(train_x), batchSize):\n            trainLoss, _, _ = sess.run([cost, probability, optimizer], feed_dict = {\n                inputT: train_x[i: i + batchSize],\n                labelT: train_y[i: i + batchSize],\n                keep_prob: keep_prob_rate\n            })\n            \n        # validation loss\n        valAcc, valLoss = sess.run([accuracy, cost], feed_dict = {\n            inputT: val_x,\n            labelT: val_y,\n            keep_prob: 1.0\n        })\n        \n        #print out\n        printResult(epoch, numberOfEpoch, trainLoss, valLoss, valAcc)\n        \n    #save\n    saver = tf.train.Saver()\n    saver.save(sess, save_dir)","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"55486340-d540-486d-8221-644d14ac42f3","_uuid":"ae69f4619dc41f64a5478e61b7e0f045ea893e02","trusted":true},"cell_type":"code","source":"# test result\n\nloaded_Graph = tf.Graph()\nwith tf.Session(graph=loaded_Graph) as sess:\n    loader = tf.train.import_meta_graph(save_dir + \".meta\")\n    loader.restore(sess, save_dir)\n    # gete tensors\n    loaded_x = loaded_Graph.get_tensor_by_name(\"input:0\")\n    loaded_y = loaded_Graph.get_tensor_by_name(\"label:0\")\n    loaded_prob = loaded_Graph.get_tensor_by_name(\"probability:0\")\n    \n    prob = sess.run(tf.argmax(loaded_prob, 1), feed_dict = {loaded_x: testData})","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"d5d297f0-509a-4782-ba9b-55099f2b8177","_uuid":"dcf5eb696733db4eed1045f8f5dc9ed484e10063","trusted":true},"cell_type":"code","source":"which = 1\nprint(\"predicted label: {}\".format(str(prob[which])))\nplotNumber(np.array([testData[which, :]]), imageSize)","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"5e696d6d-1533-499f-84d3-9719e4b41064","_uuid":"2f9482a7f0c9dd8fed68cd18c6317c8a2b67b05f","trusted":true},"cell_type":"code","source":"print(prob)","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"6eaf8f57-8909-4751-9858-053c91bbae2c","_uuid":"11bfd40f6f8b3d8edf1ed6b4e23733770a458aac","collapsed":true,"trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"import csv\noutputFile_dir = \"output.csv\"\nheader = [\"ImageID\",\"Label\"]\n\nwith open(outputFile_dir, \"w\") as csvFile:\n    writer = csv.writer(csvFile, delimiter = \",\")\n    writer.writerow(header)\n    for i, p in enumerate(prob):\n        writer.writerow([str(i + 1), str(p)])","execution_count":31,"outputs":[]},{"metadata":{"_cell_guid":"c7c35d88-80d5-46bd-b924-ca91f8665da7","_uuid":"1a9574a1c8eb38988287fb3644da5b15b4dcbcf3","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}