{"cells":[{"metadata":{"_cell_guid":"85d67daa-c866-4051-83e2-e1e95f8300b0","_uuid":"2bb174feabddf7a27a0302be8644836124aa704f"},"cell_type":"markdown","source":"TensorFow is Fun !!!\nThis is a beginner's work in both TensorFlow and Deep learning. I have used feed forward neural networks with one hidden layer."},{"metadata":{"_cell_guid":"aee2134d-cae9-401e-827c-ac3ed760c594","_uuid":"80718f1dc9801ad82f86f0d4fd69ecc78f43a505"},"cell_type":"markdown","source":"In this kernel I have used Google's amazing library TensorFlow for training a simple feed forward neural network on handwritten digits dataset."},{"metadata":{"_cell_guid":"96da08e5-b6ab-4a48-8cb7-0b758ec50a71","_uuid":"f02437f840c97fddbbcf9627846de0767132e7a5"},"cell_type":"markdown","source":"# We will start by importing the necessary libraries"},{"metadata":{"_cell_guid":"a484b584-eeb2-4b89-bad4-f9cd4c570e8e","_uuid":"24fc5e9a2c8fc744be08769b5b7737d9c4d12718","trusted":true},"cell_type":"code","source":"\nimport tensorflow as tf\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.utils import shuffle  \nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"c76d773a-a1c1-4910-907b-95465ad7c324","_uuid":"c21b3912a8e1228db6f56c7346498719f7776c01","collapsed":true,"trusted":true},"cell_type":"code","source":"# Let's import the test dataset\ntest = pd.read_csv('../input/test.csv')\ntrain = pd.read_csv(\"../input/train.csv\")","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"79616035-1588-41b1-bb5e-8b033d6540ad","_uuid":"05d2d3c4f48cc19083f30b079604d9650bd63ccd","collapsed":true,"trusted":true},"cell_type":"code","source":"# Import the training dataset by making a function read_dataset.\ndef read_dataset():\n    train = pd.read_csv(\"../input/train.csv\")\n    \n    # In X we will have all the values of the pixels.\n    # Y will have the label that is what number is it.\n    X = train[train.columns[1:785]].values # features\n    y = train[train.columns[0]] # labels\n   \n\n    # We need to encode our dataset using simple encoding.\n    encoder = LabelEncoder()\n    encoder.fit(y)\n    y = encoder.transform(y)\n    Y = one_hot_encode(y)\n    return(X,Y)\n","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"13dccdfa-b2eb-4cd0-9abb-4129cd8710c6","_uuid":"24039b178099c967e988c61ac6f3738c21aeca27","collapsed":true,"trusted":true},"cell_type":"code","source":"# Using one hot encoding.\ndef one_hot_encode(labels):\n    n_labels = len(labels)\n    n_unique_labels = len(np.unique(labels))\n    one_hot_encode = np.zeros((n_labels, n_unique_labels))\n    one_hot_encode[np.arange(n_labels), labels] = 1\n    return one_hot_encode\n\n","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"06031a0c-0921-4692-97d2-ad8427a2af4a","_uuid":"c39547c2981d141ba2290d7ed981958d49fcb550","trusted":true},"cell_type":"code","source":"X, Y = read_dataset()\n\n# Shuffle the dataset to mix up the rows.\nX, Y = shuffle(X, Y, random_state=1)\n\n# Spliiting the dataset into train and test with 20% being the test size and 80% is the training size.\ntrain_x, test_x, train_y, test_y = train_test_split(X, Y, test_size=0.20, random_state=415)\n\n# The rate by which our dataset will learn.\nlearning_rate = 0.01\n\n# Epochs is basically the number of iterations.\ntraining_epochs = 20\ncost_history = np.empty(shape=[1], dtype=float)\nn_dim = X.shape[1]\nprint(\"n_dim\", n_dim)\nn_class = 10\n","execution_count":17,"outputs":[]},{"metadata":{"_cell_guid":"e40ce0da-2af3-435a-b0f3-dec86fe8bbc0","_uuid":"ff4411ff2be6863b1370424e2040bed4c59006fc","collapsed":true,"trusted":true},"cell_type":"code","source":"# I have used  one hidden layer with the number of neurons equal to 650.\nn_hidden_1 = 650\n","execution_count":18,"outputs":[]},{"metadata":{"_cell_guid":"b377d291-5f15-40cd-bdeb-4b38360613cb","_uuid":"3050471778c1f18e21c3044551675dc49ed1ae1e","collapsed":true,"trusted":true},"cell_type":"code","source":"batch_size = 128","execution_count":19,"outputs":[]},{"metadata":{"_cell_guid":"fe6b7072-203b-4092-8c20-15b26d118930","_uuid":"f54342c39107ad5aa86825bf710d4a4c3371168a"},"cell_type":"markdown","source":"# Using TensorFlow"},{"metadata":{"_cell_guid":"93a63806-c29a-4eb9-b111-a1cdf7e5d722","_uuid":"f2e727ebc957bf30809d90347b39836511f3a920"},"cell_type":"markdown","source":"**x and y_ are placeholders whereas weights and biases will be variables.**"},{"metadata":{"_cell_guid":"f018d50e-1501-497d-a1a7-36ddb27c8092","_uuid":"753711d60073934bc8caca8cf9e610833263f716","collapsed":true,"trusted":true},"cell_type":"code","source":"x = tf.placeholder(tf.float32,[None, n_dim])\nW = tf.Variable(tf.zeros([n_dim, n_class])) # Weights\nb = tf.Variable(tf.zeros([n_class])) # Biases\ny_ = tf.placeholder(tf.float32,[None,n_class])\n","execution_count":20,"outputs":[]},{"metadata":{"_cell_guid":"68b4877f-187d-43d8-ad24-c60d56cb0b8c","_uuid":"6ba8ebcefef1b4765747a11e11d10d00df842f1c"},"cell_type":"markdown","source":"**Using Sigmoid Activation for hidden layer and linear activation for output layer**"},{"metadata":{"_cell_guid":"c13c2afa-4b97-418f-88a9-6911331b299c","_uuid":"bca2f891c23a7b345a28bed61718b40f7af299c3","collapsed":true,"trusted":true},"cell_type":"code","source":"def multilayer_perceptron(x, weights, biases):\n\n    # Hidden layer with sigmoid activation\n    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])\n    layer_1 = tf.nn.sigmoid(layer_1)\n\n   # Output layer with linear activation\n    out_layer = tf.matmul(layer_1, weights['out']) + biases['out']\n    return out_layer\n","execution_count":21,"outputs":[]},{"metadata":{"_cell_guid":"e75f235a-8bca-4aa9-a9d6-52875a999f3e","_uuid":"29d33206af098ff37d6d7f1ea99d05532ea19b2a"},"cell_type":"markdown","source":"**Making dictionary for the weights and biases**"},{"metadata":{"_cell_guid":"81e54ca8-51e3-4d16-ad13-f453ba06fee8","_uuid":"7aaa76b2a13a997a4711fb50e4caa514d05a8b59","collapsed":true,"trusted":true},"cell_type":"code","source":"weights = {\n    'h1': tf.Variable(tf.truncated_normal([n_dim, n_hidden_1])),\n    'out': tf.Variable(tf.truncated_normal([n_hidden_1, n_class]))\n}\nbiases = {\n    'b1': tf.Variable(tf.truncated_normal([n_hidden_1])),\n    'out': tf.Variable(tf.truncated_normal([n_class]))\n}\n","execution_count":22,"outputs":[]},{"metadata":{"_cell_guid":"00efa7fa-18d7-4036-8881-df229c014249","_uuid":"8cb83f6764f8022b2084e8e665b8e579f5fbae6c","collapsed":true,"trusted":true},"cell_type":"code","source":"# To initialize the variables\ninit = tf.global_variables_initializer()\ny = multilayer_perceptron(x, weights, biases)","execution_count":23,"outputs":[]},{"metadata":{"_cell_guid":"b73aaee0-c0e7-493f-8774-18286052ee8f","_uuid":"619d8567c8c9dbebca4af22b2787a1eacd2ba50d"},"cell_type":"markdown","source":"** Using gradient descent optimizer**"},{"metadata":{"_cell_guid":"93e8170a-f430-410e-a3f9-bfd43909457d","_uuid":"e82a9ea20117432a6f6f4a50ceb57cd50a5b6af6","trusted":true,"collapsed":true},"cell_type":"code","source":"cost_function = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_))\ntraining_step = tf.train.AdamOptimizer(learning_rate).minimize(cost_function)\n\nsess = tf.Session()\nsess.run(init)\n","execution_count":24,"outputs":[]},{"metadata":{"_cell_guid":"427edc8b-3791-427e-89f7-5a25a495d8ec","_uuid":"64a994a0b6914227c7cc149dee5cabe02ef3872e"},"cell_type":"markdown","source":"We are running the model for 20 epochs. Epoch basically means an iteration."},{"metadata":{"_cell_guid":"1140ea58-2b26-4ce8-b452-55075693291a","_uuid":"39a5af3d35f3cd91dbeb30211fde4d4ad743deef","trusted":true,"collapsed":true},"cell_type":"code","source":"# Calculate the cost and the accuracy for each epoch\n\n# List for mean square error.\nmse_history = []\n\n# List for accuracy.\naccuracy_history = []\n\nfor epoch in range(training_epochs):\n    \n    \n    total_batch = int(train.shape[0]/batch_size)\n    for i in range(total_batch):\n            avg_cost = 0\n            #batch_x, batch_y = batch_creator(batch_size, train_x.shape[0], 'train')\n            sess.run(training_step, feed_dict={x: train_x, y_: train_y})\n            cost = sess.run(cost_function, feed_dict={x: train_x, y_: train_y})\n            \n            avg_cost += cost/total_batch\n            cost_history = np.append(cost_history, cost)\n            \n    \n    # Calculating the number of correct predictions.\n    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n    \n    # Calculating the accuracy.\n    print(\"Accuracy: \", (sess.run(accuracy, feed_dict={x: test_x, y_: test_y})))\n    pred_y = sess.run(y, feed_dict={x: test_x})\n    \n    # Calculating the mean square error.\n    mse = tf.reduce_mean(tf.square(pred_y - test_y))\n    mse_ = sess.run(mse)\n    mse_history.append(mse_)\n    accuracy = (sess.run(accuracy, feed_dict={x: train_x, y_: train_y}))\n    accuracy_history.append(accuracy)\n\n    print('epoch : ', epoch, ' - ', 'cost: ', cost, \" - MSE: \", mse_, \"- Train Accuracy: \", accuracy)\n\n#save_path = saver.save(sess, model_path)\n#print(\"Model saved in file: %s\" % save_path)\n\n# Plot mse and accuracy graph\n\nplt.plot(mse_history, 'r')\nplt.show()\nplot.title(\"Mean Square Error\")\nplt.plot(accuracy_history)\nplot.title(\"Accuracy\")\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"89e005a7-d67d-49b4-8832-b30991aaf9ed","_uuid":"615225c59e5ab813104e6db346c10cadc4405286","collapsed":true,"trusted":true},"cell_type":"code","source":"# Print the final accuracy\ncorrect_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\nprint(\"Test Accuracy: \", (sess.run(accuracy, feed_dict={x: test_x, y_: test_y})))\n\n# Print the final mean square error\n\npred_y = sess.run(y, feed_dict={x: test_x})\nmse = tf.reduce_mean(tf.square(pred_y - test_y))\nprint(\"MSE: %.4f\" % sess.run(mse))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6c87a060-ab20-4a01-b917-c2b9d3858cdc","_uuid":"811dfbae3c7f5589c27f6d20d0e90c0b17133655"},"cell_type":"markdown","source":"# Applying on the validation set."},{"metadata":{"_cell_guid":"a6264bb7-5057-42f6-957e-c5817cfae04d","_uuid":"78ca88ee022c71d28011af9ed7bc2297692669bd","collapsed":true,"trusted":true},"cell_type":"code","source":"test_x  = test.iloc[0:28000].values\npredict = tf.argmax(output_layer, 1)\npred = predict.eval({x: test_x.reshape(-1, input_num_units)})\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c59da860-08f5-47e4-b604-ac77d29717af","_uuid":"0fc09c97039362a893798ba6e2abff1ce158cc49","collapsed":true,"trusted":true},"cell_type":"code","source":"df = pd.DataFrame({'Label': pred})\n# Add 'ImageId' column\ndf1 = pd.concat([pd.Series(range(1,28001), name='ImageId'), \n                              df[['Label']]], axis=1)\n\n\ndf1.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"93c61687-1630-45e8-9c77-d6e8e8490922","_uuid":"3cdfd4f5dbcc3c5556dbbeead439e4fca14125ab"},"cell_type":"markdown","source":"Any suggestions are welcomed.  If there are any mistakes please do inform. If you like the kernel please do cast an upvote.\n\n\n*Ayushi Asthana\nMarch 2018*"},{"metadata":{"_cell_guid":"87984af5-17e1-41d5-a483-0df26a81c0bd","_uuid":"b6cc10795cb5cb9ef3950364e4ff2064d8dc9cf6","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}