{"cells":[{"metadata":{"_uuid":"87f955f3bb440acc8805d0a9241be802252c3dff","_cell_guid":"6723d17f-bb6d-46e5-a085-2627ea45173b"},"cell_type":"markdown","source":"# Convolution Neural Network\nThis notebook is quick CNN (convolution neual network) predicting digit for [kaggle competition](https://www.kaggle.com/c/digit-recognizer). \n\nPlease consider follow before reading this notebook.\n- We don't verify data in this notebook.\n- I would like to make this notebook simple enough for anyone who new to CNN can follow the code easily.\n- I also provide MLP (multilayer perceptron) code with the same problem. Please take a look if you new to Deep Learning."},{"metadata":{"_uuid":"90723df69bb1ae9d2b01707f9210093f8b4033ea","_cell_guid":"d77774d4-bded-4bd0-8389-2c2ae84971f4"},"cell_type":"markdown","source":"## 1. Load data"},{"metadata":{"_uuid":"db3bc022745f57afe73b779bf12dafd24196d3a4","collapsed":true,"_cell_guid":"1c93984c-6073-46f3-9ea7-00b5a4a2f84a","trusted":false},"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nimport numpy as np\n\ntrain_dataset = pd.read_csv('../input/train.csv')\ntest_dataset = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77c4eebeaa24e7190e3d3b62a248f9cc2fb810f3","_cell_guid":"d7aa33da-dfd7-4681-9a3c-451a387251f3","trusted":false,"collapsed":true},"cell_type":"code","source":"print(\"Train Dataset\")\nprint(train_dataset[:5])\nprint(\"Test Dataset\")\nprint(test_dataset[:5])\nprint(\"Train Size : {}\".format(train_dataset.shape[0]))\nprint(\"Test Size : {}\".format(test_dataset.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f85b15d86abc43864168c5713631bba370e60007","_cell_guid":"27c541c5-4401-4f2f-a7f1-4c2e89d5df75"},"cell_type":"markdown","source":"## 2. Prepare Data\nBefore training we need to prepare data a little bit. here is the detail\n- We need to scale each pixel to the range of 0 - 1\n- Since Kaggle don't have validate data for us,so we have to split some from training data.\n- We need to spearate label (train_y) out of feature (train_x) so we can feed it to our network as label and feature. "},{"metadata":{"_uuid":"2eb1e7f6becb2007af59cb2319b08f3e4e8172c5","_cell_guid":"8e0fd373-98d5-418a-aac7-41fd0a614a6c","trusted":false,"collapsed":true},"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import LabelBinarizer\n\nvalidation_ratio = 0.1\n\npixel_scaler = MinMaxScaler()\none_hot = LabelBinarizer()\none_hot.fit(range(10))\ntrain_y = one_hot.transform(train_dataset['label'])\ntrain_x = train_dataset.drop('label', axis=1)\nprint(train_x.values.shape)\ntrain_x = pixel_scaler.fit_transform(train_x)\ntrain_x = train_x.reshape( (-1, 28, 28, 1) )\ntest_x = pixel_scaler.transform(test_dataset)\ntest_x = test_x.reshape( (-1, 28, 28, 1) )\n\nsplit = int(train_dataset.shape[0] * (1 - validation_ratio))\n\nvalid_x = train_x[split:train_x.shape[0]]\nvalid_y = train_y[split:train_x.shape[0]]\ntrain_x = train_x[0:split]\ntrain_y = train_y[0:split]\nprint(\"Train Size : {}\".format(train_x.shape[0]))\nprint(\"Validation Size : {}\".format(valid_x.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d7dc35a32f6bce798b11dfa183b02b43cb89255","_cell_guid":"a3f86be3-45f0-4c86-816c-0bc6cc1928d0"},"cell_type":"markdown","source":"## 3. Create Model\nBelow is the model code.\n- Initialize DigitalRecognizerCNN at first.\n- Call make_nn() to set up tensorflow graph.\n- Call train() for train network after training the network has been stored and ready for prediction.\n- Call predict() for make a prediction with the latest saved network."},{"metadata":{"_uuid":"08d5378401f0a79f920de424f244780684962411","collapsed":true,"_cell_guid":"ffa2d4fa-4a36-4305-a353-74d18d9329aa","trusted":false},"cell_type":"code","source":"class DigitRecognizerCNN(object):\n    def __init__(self, width=28, height=28, channel=1, label_size=10):\n        self.width = width\n        self.height = height\n        self.channel = channel\n        self.label_size = label_size\n    \n    def build_input(self):\n        model_x = tf.placeholder(tf.float32, [None, self.width, self.height, self.channel])\n        model_y = tf.placeholder(tf.int32, [None, self.label_size])\n        model_lr = tf.placeholder(tf.float32)\n        return model_x, model_y, model_lr\n    \n    def build_cnn(self, model_x, model_y):\n        model_x = tf.reshape(model_x, (-1, 28, 28, 1))\n        \n        conv_1 = tf.layers.conv2d(model_x, 3, 16, padding='same', activation=tf.nn.relu)\n        pooling_1 = tf.layers.max_pooling2d(conv_1, 2, 1, padding='same')\n        conv_2 = tf.layers.conv2d(pooling_1, 3, 32, padding='same', activation=tf.nn.relu)\n        pooling_2 = tf.layers.max_pooling2d(conv_2, 2, 1, padding='same')\n        flatten = tf.layers.flatten(pooling_2)\n        logits = tf.layers.dense(flatten, self.label_size, activation=None)\n        return logits\n    \n    def build_output(self, model_y, logits):\n        output = tf.argmax(logits, axis=1)\n        loss = tf.losses.softmax_cross_entropy(model_y, logits)\n        accuracy = tf.reduce_mean( tf.cast( tf.equal( tf.argmax(model_y, axis=1), output ), tf.float32) )\n        return output, loss, accuracy\n    \n    def model_opt(self, model_lr, loss):\n        opt = tf.train.AdamOptimizer(model_lr).minimize(loss)\n        return opt\n    \n    def make_nn(self):\n        tf.reset_default_graph()\n        self.model_x, self.model_y, self.model_lr = self.build_input()\n        self.logits = self.build_cnn(self.model_x, self.model_y)\n        self.output, self.loss, self.accuracy = self.build_output(self.model_y, self.logits)\n        self.opt = self.model_opt(self.model_lr, self.loss)\n        \n    def train(self, epoch, learning_rate, train_x, train_y, batch_size, valid_x, valid_y, get_batch_func, print_every):\n        t_loss_list = []\n        v_loss_list = []\n        accuracy_list = []\n        saver = tf.train.Saver()\n        with tf.Session() as sess :\n            sess.run(tf.global_variables_initializer())\n            counter = 0\n            for e in range(epoch) :\n                for x,y in get_batch_func(train_x, train_y, batch_size):\n                    feed_dict = {self.model_x:x, self.model_y:y, self.model_lr:learning_rate}\n                    loss, _ = sess.run([self.loss, self.opt], feed_dict=feed_dict)\n                    if counter % print_every == 0 :\n                        feed_dict = {self.model_x:valid_x, self.model_y:valid_y, self.model_lr:learning_rate}\n                        v_loss, accuracy = sess.run([self.loss, self.accuracy], feed_dict=feed_dict)\n                        print(\"Epoch: {}/{}, Step: {}, T Loss: {:.4f}, V Loss: {:.4f}, Accuracy: {:.4f}\".format(e+1, \n                                    epoch, counter, loss, v_loss, accuracy))\n                        t_loss_list.append( loss )\n                        v_loss_list.append( v_loss )\n                        accuracy_list.append( accuracy )\n                        save_path = saver.save(sess, \"tensor/model.ckpt\")\n                    counter += 1\n        return t_loss_list, v_loss_list, accuracy_list\n    \n    def predict(self, predict_x):\n        saver = tf.train.Saver()\n        with tf.Session() as sess:\n            sess.run(tf.global_variables_initializer())\n            saver.restore(sess, \"tensor/model.ckpt\")\n            feed_dict = {self.model_x:predict_x}\n            output = sess.run(self.output, feed_dict=feed_dict)\n            return output","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"786b95b60616fea8ec2e09735f4daaa28a6346cf","collapsed":true,"_cell_guid":"2c123b21-0d77-41bb-817e-0d21eda1b882","trusted":false},"cell_type":"code","source":"# For Testing --- Delete after use\ndr_cnn = DigitRecognizerCNN()\ndr_cnn.make_nn()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0e98430d894bbe40d0d535ee6dc8066a0b9ba6df","collapsed":true,"_cell_guid":"fd4c7c7c-da26-4bed-a48f-2451c530daa8","trusted":false},"cell_type":"code","source":"def get_batch(feature, label, batch_size):\n    num_batch = feature.shape[0] // batch_size\n    for i in range(num_batch):\n        start = i * batch_size\n        end = (i + 1) * batch_size\n        x = feature[start:end]\n        y = label[start:end]\n        yield x, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0606739bcecd7cf95b4ae1bb846de53b86d8c14","_cell_guid":"31c4dd3e-6bd8-400d-889a-0b1339638711"},"cell_type":"markdown","source":"## 4. Train"},{"metadata":{"_cell_guid":"93d51e0f-9347-419d-ba2c-18ca21ed49e9","scrolled":false,"_uuid":"f48cc23de156e4eb8189e331782d1f63065e5e75","trusted":false,"collapsed":true},"cell_type":"code","source":"learning_rate = 0.0001\nbatch_size = 32\nepoch = 20\nprint_every = 1000\n\ncnn = DigitRecognizerCNN()\ncnn.make_nn()\nt_loss, v_loss, acc = cnn.train(epoch, learning_rate, train_x, train_y, batch_size, valid_x, valid_y, get_batch, print_every)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3834cb95cd78fe9470fdc8660d08455e7adfebaf","_cell_guid":"4afbf9e1-5781-4bd1-a84a-cc3bfe7e0a41"},"cell_type":"markdown","source":"## 5. Predict"},{"metadata":{"_uuid":"e3ae88f5234d6252d8eefd25b887641d8f50bf8b","_cell_guid":"b839b331-f491-4011-bc5d-58e012fb9ae6","trusted":false,"collapsed":true},"cell_type":"code","source":"cnn = DigitRecognizerCNN()\ncnn.make_nn()\nsize = 2048\nnum_batch = len(test_x) // size + 1\npredict = []\nfor i in range(num_batch):\n    p = cnn.predict(test_x[i * size: (i+1) * size])\n    predict = np.concatenate([predict, p])\nprint(predict.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"153fd60ed2c52be160173ea9a7672302af000bb4","_cell_guid":"1abffbb0-7252-43f4-ba3d-bb5ac5055dfa"},"cell_type":"markdown","source":"## 6. Make CSV\nThis is just making csv file as kaggle required for submittion."},{"metadata":{"_uuid":"96eb7f0d46ad6bd8f3f262645c2cfe380e2dbd00","collapsed":true,"_cell_guid":"e66278db-3f82-4510-8a34-583b0c0b4369","trusted":false},"cell_type":"code","source":"import csv\n\nwith open('predict.csv', 'w') as csv_file :\n    writer = csv.writer(csv_file)\n    writer.writerow(['ImageId', 'Label'])\n    for i in range(len(predict)) :\n        writer.writerow([str(i+1), str(int(predict[i]))])\n    csv_file.close()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dbc5ca56b18b23acea0d0a43ec36e5ac1ecd3f93","collapsed":true,"_cell_guid":"7b1255ec-c463-4cba-9b7b-a5674197576f","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}