{"cells":[{"metadata":{"_uuid":"f6845bea83d51e604fb1d499afc4f4ac01870107"},"cell_type":"markdown","source":"# LeNet5 Architecture\n#### Gradient Based Learning Applied to Document Recognition, 1998"},{"metadata":{"_uuid":"58325a9302a3e8a9524f9b05b90100c3667437fa"},"cell_type":"markdown","source":"### INPUT -> CONV -> AVG POOL -> CONV -> AVG POOL -> FC -> FC ->FC"},{"metadata":{"_uuid":"48680fae1aaedbfee4bb7a77040d1e4a119e017c"},"cell_type":"markdown","source":"# Import"},{"metadata":{"trusted":true,"_uuid":"0948a5a1f4c2f875fa8a7b1dc70d034ca409917a"},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom tensorflow.contrib.layers import flatten","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"564d69eef84cc9b49b82da91e0fc5938f2cd6370"},"cell_type":"markdown","source":"# Read Input"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_X = pd.read_csv(\"../input/train.csv\")\ntrain_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"97ea1dfa99e3f00696c1a56c5f90a5db7e2009cc"},"cell_type":"code","source":"train_Y = train_X['label'].values\ntrain_X = train_X.drop('label',axis=1).values\ntrain_X.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8953a903135a5fef7c92a9191df48a017c8f3318"},"cell_type":"markdown","source":"# Pre-processing"},{"metadata":{"trusted":true,"_uuid":"e88f89c0959b41724821189c4274b9aad0336fe8"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x, test_x, train_y, test_y = train_test_split(train_X, train_Y, test_size = 0.3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5faf1e60898a191a6acc35621cceea9fe1819c2"},"cell_type":"code","source":"train_x = train_x/255.0  #normalization of training set\ntest_x = test_x/255.0  #normalization of test set\n\ntrain_x = train_x.reshape(-1,28,28,1)\ntest_x = test_x.reshape(-1,28,28,1)\n\ntrain_y = np.eye(10)[train_y]  #one hot encoding\ntest_y = np.eye(10)[test_y]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c03fc25c400dcf221664026364701fc81868d000"},"cell_type":"code","source":"#pad image with zeros\ntrain_x      = np.pad(train_x, ((0,0),(2,2),(2,2),(0,0)), 'constant')\ntest_x       = np.pad(test_x, ((0,0),(2,2),(2,2),(0,0)), 'constant')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4396ebbb8e6e612cbddc5418b0976c2e02aa6a26"},"cell_type":"code","source":"print('Shape of train_x \\t: ', train_x.shape)\nprint('Shape of train_y \\t: ', train_y.shape)\nprint('Shape of test_x  \\t: ', test_x.shape)\nprint('Shape of test_y  \\t: ', test_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab915972c9797d0c41d66e53e1ae0e24487dccfe"},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true,"_uuid":"2e740f5bca1a8e62c6ef31a0a158bca558ab9ad7"},"cell_type":"code","source":"def batch(batch_size, x, y):\n    index = np.arange(len(x))\n    np.random.shuffle(index)\n    \n    b_index = index[0:batch_size]\n    \n    x_next = [x[i] for i in b_index]\n    y_next = [y[i] for i in b_index]\n    \n    x_next = np.asarray(x_next)\n    y_next = np.asarray(y_next)\n    return x_next,y_next","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ffc8df9d172550e90660a9e5fac6e2b548dda00"},"cell_type":"code","source":"def conv_layer(x, w, b, stride=1, padding='VALID'):\n    x = tf.nn.conv2d(x,w,strides=[1,stride,stride,1], padding = padding)\n    x = tf.nn.bias_add(x,b)\n    x = tf.nn.tanh(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"32333a49a7d7413add6a2de6ef3eb08736d52a44"},"cell_type":"code","source":"def pool_layer(x, f=2, stride=2, padding = 'VALID'):\n    x = tf.nn.avg_pool(x, ksize=[1,f,f,1], strides=[1,stride,stride,1], padding=padding)\n    x = tf.nn.tanh(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b52f5ba6bf7e8e68099a653f078be3fa009f1f4"},"cell_type":"code","source":"def fc_layer(x,w,b):\n    x = tf.matmul(x,w)\n    x = tf.nn.bias_add(x,b)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9437292eca1ec15087732a888905a7cf518122a"},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true,"_uuid":"adbba842004994d1fa8f8616f0949b9d0f89ada2"},"cell_type":"code","source":"# hyperparameters\nlearning_rate = 0.001\nbatch_size = 128\nepoch = 10000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11597f08e360b17a97a0852fc8d82aa8f7ceec17"},"cell_type":"code","source":"# image dimensions and number of classes\nimage_width = 32\nimage_height = 32\ncolor_channels = 1\nn_classes = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2b67c6093272cb2ee9f1dd6c5558235dc9bf224"},"cell_type":"code","source":"# placeholders\nX = tf.placeholder(tf.float32, shape = [None, image_width, image_height, color_channels]) \nY = tf.placeholder(tf.float32, shape = [None, n_classes]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4bb26e5b8a7886d33a7b439e7b5142e1e04b211"},"cell_type":"code","source":"# parameters\nweights =  { 'w1' : tf.Variable(tf.truncated_normal([5,5,1,6], stddev=0.1)),\n             'w2' : tf.Variable(tf.truncated_normal([5,5,6,16], stddev=0.1)),\n             'w3' : tf.Variable(tf.truncated_normal([400,120], stddev=0.1)),\n             'w4' : tf.Variable(tf.truncated_normal([120,84], stddev=0.1)),\n             'w5' : tf.Variable(tf.truncated_normal([84,10], stddev=0.1))}\n            \nbiases = {'b1' : tf.Variable(tf.truncated_normal([6], stddev=0.1)),\n          'b2' : tf.Variable(tf.truncated_normal([16], stddev=0.1)),\n          'b3' : tf.Variable(tf.truncated_normal([120], stddev=0.1)),\n          'b4' : tf.Variable(tf.truncated_normal([84], stddev=0.1)),\n          'b5' : tf.Variable(tf.truncated_normal([10], stddev=0.1))}         ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0659e34802fcc1cd36a0312d1e44fbeec4ce6d1d"},"cell_type":"code","source":"def Network(x,weights,biases):\n    conv1 = conv_layer(x,weights['w1'],biases['b1']) # input - 32x32x1, output - 28x28x6\n    pool1 = pool_layer(conv1) # input - 28x28x16, output - 14x14x6\n    conv2 = conv_layer(pool1,weights['w2'],biases['b2']) # input - 14x14x6, output - 10x10x16\n    pool2 = pool_layer(conv2) # input - 10x10x16, output - 5x5x16\n    \n    flat = flatten(pool2) # input - 5x5x16, output - 400\n    \n    fc1 = fc_layer(flat,weights['w3'],biases['b3']) # input - 400, output - 120\n    fc1 = tf.nn.tanh(fc1)\n    fc2 = fc_layer(fc1,weights['w4'],biases['b4']) # input - 120, output - 84\n    fc2 = tf.nn.tanh(fc2)\n    fc3 = fc_layer(fc2,weights['w5'],biases['b5']) # input - 84, output - 10\n    \n    return fc3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"811844e7b2ffdf1a02fcdb30ebe9a0af401836f5"},"cell_type":"code","source":"logits = Network(X,weights,biases) #model output\n\n# softmax with cross entropy loss function\nloss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = Y))\noptimizer = tf.train.AdamOptimizer(learning_rate = learning_rate) # optimizer\ntrain = optimizer.minimize(loss) # minimize loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"634ebd381e1b1939dabe4bd1ee1fe7448a8be22c"},"cell_type":"code","source":"correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"424d06cdca6ffc2505fc01c0ec248a66e4cca953"},"cell_type":"code","source":"init = tf.global_variables_initializer()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"48c06485f167a410d3e2d188dd26bbb0c963e443"},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true,"_uuid":"89ea543d743c61eb663667473baf2e9c83ac1bff"},"cell_type":"code","source":"with tf.Session() as sess:\n    \n    sess.run(init)\n    \n    cost_hist, acc_hist = [], []\n    \n    for epoch in range(1, epoch + 1):\n        \n        batch_x, batch_y = batch(batch_size, train_x, train_y)\n        \n        sess.run(train, feed_dict = { X : batch_x, Y : batch_y})\n    \n        if epoch % 500 == 0:\n            c, acc = sess.run([loss, accuracy], feed_dict = { X : batch_x, Y : batch_y})\n            cost_hist.append(c)\n            acc_hist.append(acc)\n            print('Epoch ' + str(epoch) + ', Cost: ' + str(c) + ', Accuracy: ' + str(acc))\n\n    W = sess.run(weights)\n    B = sess.run(biases)\n    print('-' * 70)\n    print('\\nOptimization Finished\\n')\n    print('Accuracy on train data \\t: ' + str(sess.run(accuracy, feed_dict = { X : train_x, Y :train_y}) * 100) + ' %')\n    print('Accuracy on test data  \\t: ' + str(sess.run(accuracy, feed_dict = { X : test_x, Y : test_y}) * 100) + ' %')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34b971fc5c888a39816786fba0c991face7de444"},"cell_type":"markdown","source":"# Cost Function Variation"},{"metadata":{"trusted":true,"_uuid":"f2bb7ef405b6dbdbdab2178f2fe79cc7bc212f02"},"cell_type":"code","source":"eps = list(range(500, epoch+500, 500))\nplt.plot(eps, cost_hist)\nplt.title(\"Change in cost\")\nplt.ylabel('Cost')\nplt.xlabel('Epoch')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d12d44bf5b3d5eb3038fa79fb9d982cac94a8d2a"},"cell_type":"markdown","source":"# Accuracy Variation"},{"metadata":{"trusted":true,"_uuid":"5191b3666e45cbaf87a9c98d7150949e17661d4a"},"cell_type":"code","source":"eps = list(range(500, epoch+500, 500))\nplt.plot(eps, acc_hist)\nplt.title(\"Change in accuracy\")\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b41901a23658d657ccbe11a9ee8c8152e6a8f3ff"},"cell_type":"markdown","source":"# Prediction"},{"metadata":{"trusted":true,"_uuid":"54f24d64d846ec48ef2a091121744a534b3449de"},"cell_type":"code","source":"test = pd.read_csv(\"../input/test.csv\")\n\n#set parameters\nfor key in weights.keys():\n    weights[key] = tf.Variable(W[key])\nfor key in biases.keys():\n    biases[key] = tf.Variable(B[key])\n    \nx = test.values.reshape(-1,28,28,1) # reshape\nx = x/255.0 # normalize\nx = np.pad(x, ((0,0),(2,2),(2,2),(0,0)), 'constant') # pad with zeros, output - 32x32x1 images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb9992e02e473dd9d27ba5d725296b027ba75fed"},"cell_type":"code","source":"logits = Network(X, weights, biases)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60cbdf07d08914e97a1df5e30fd332718eb88101"},"cell_type":"code","source":"with tf.Session() as sess:\n    \n    for key in weights.keys():\n        sess.run(weights[key].initializer)\n    for key in biases.keys():\n        sess.run(biases[key].initializer)\n    \n    output = sess.run(logits, feed_dict= {X : x}) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b03c05fd975986ce3959d4de8497e6f3c2ff8c1"},"cell_type":"code","source":"predictions = np.argmax(output, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a10be45349a3095e09d85d29c94653b11b0277a7"},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_uuid":"893941b9d95e00bbae5ab6245f298d4aae74998a"},"cell_type":"code","source":"index = test.index +1 \n\nImageId = list(index)\nLabel = list(predictions)\n\ndf = pd.DataFrame([ImageId ,Label])\ndf = df.T\ndf.columns = [['ImageId', 'Label']]\n\ndf.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f469d490b32c091cf8e9ead4e9def7b3c23ae1a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}