{"cells": [{"metadata": {"_execution_state": "idle", "_uuid": "82c9640760c987553f48f561c24536a9ce1cf0a8", "_cell_guid": "efa4f0f2-76e1-46ce-ab1c-8d0ae1ff8ac6"}, "cell_type": "markdown", "source": ["# Digit Recognizer with TensorFlow\n", "\n", "In this kernel I'm trying to solve Kaggle's \"Digit Recognizer\" challenge.\n", "\n", "## Loading modules"]}, {"metadata": {"_uuid": "bb805fdd30a52eb2263bd9511efab044e3e7c372", "_cell_guid": "3482ab6b-15d7-4df7-a499-33fe6e1d38e7", "collapsed": true}, "source": ["import tensorflow as tf\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline"], "cell_type": "code", "outputs": [], "execution_count": 2}, {"metadata": {"_uuid": "ea6abcd8c8860bb144a72d02c2d1d02d7bc0e6de", "_cell_guid": "ef65a0ae-a9ae-481c-94da-d325f5b49539", "collapsed": true}, "cell_type": "markdown", "source": ["## Loading datasets\n", "\n", "We'll divide the dataset from the `train.csv` file into train, dev, and test, in proportions 8:1:1"]}, {"metadata": {"_uuid": "5d6986f6e04f3e97d47faae0467d5d88450d9af2", "_cell_guid": "e6156f1f-1bef-485f-ba54-46dd47edb06d", "collapsed": true}, "source": ["raw_dataset = pd.read_csv('../input/train.csv').values\n", "np.random.shuffle(raw_dataset)\n", "all_labels = raw_dataset[:, 0]\n", "all_features = raw_dataset[:, 1:] / 255\n", "\n", "num_examples = all_labels.size\n", "a, b = num_examples * 8 // 10, num_examples * 9 // 10\n", "\n", "dataset = {\n", "    'train_labels': all_labels[:a],\n", "    'dev_labels': all_labels[a:b],\n", "    'test_labels': all_labels[b:],\n", "    \n", "    'train_features': all_features[:a, :],\n", "    'dev_features': all_features[a:b, :],\n", "    'test_features': all_features[b:, :]\n", "}"], "cell_type": "code", "outputs": [], "execution_count": 3}, {"metadata": {"_uuid": "53be93e920595a6df427a7e37c3664177883e508", "_cell_guid": "e985e9e6-11f5-4701-a367-4404e6a20e85"}, "cell_type": "markdown", "source": ["## Plotting sample digit"]}, {"metadata": {"_uuid": "35bd07e557088cf432833e8189ae37b9c6bdb48c", "_cell_guid": "0a5c2c54-4064-4d37-917f-98b72513c5ed"}, "source": ["def plot_digit(pixels):\n", "    plt.imshow(pixels.reshape(28, 28))\n", "    plt.show()\n", "\n", "plot_digit(dataset['test_features'][0, :])"], "cell_type": "code", "outputs": [], "execution_count": 4}, {"metadata": {"_uuid": "863c63ae5cc14b454b46ad9de86bc81d3e1a9b91", "_cell_guid": "4644477e-0bd2-4e7e-ab83-4c731ebc58a6"}, "cell_type": "markdown", "source": ["## First model\n", "- one hidden layer (ReLU, 200 units)\n", "- softmax on the output layer\n", "- no regularisation\n", "- Adam optimizer\n", "- full-batch training"]}, {"metadata": {"_uuid": "ce7142a5064c215c28c69a5291045c4e7a09d858", "_cell_guid": "08e038d4-ac37-4d04-ae1f-d4d1000e4596"}, "cell_type": "markdown", "source": ["### Input nodes\n", "The following code creates placeholders for features and labels, as well as labels encoded with \"one-hot\" (used in cost calculation):"]}, {"metadata": {"_uuid": "391dabcb85ec8368e0fb4728bc260501678a86cb", "_cell_guid": "ac8ecf71-cc66-45bb-b8b3-7f465c3473b5", "collapsed": true}, "source": ["def input_nodes(num_features, num_labels):\n", "    X = tf.placeholder(tf.float32, name = 'X', shape = [None, num_features])\n", "    labels = tf.placeholder(tf.int64, name = 'labels', shape = [None])\n", "    with tf.name_scope('labels_to_Y'):\n", "        Y = tf.one_hot(labels, num_labels, name = 'Y')\n", "    return X, labels, Y"], "cell_type": "code", "outputs": [], "execution_count": 5}, {"metadata": {"_uuid": "bee3d16299b72b7065d40333ac4874b3ea8ea7f3", "_cell_guid": "ff33b934-a61e-424a-8bdc-b4b1697e858b"}, "cell_type": "markdown", "source": ["### Linear layer"]}, {"metadata": {"_uuid": "bcc91d9fec47f88498187371f3e4a76bc33d3535", "_cell_guid": "ecd3160a-cf4f-48f2-af37-40c3aeb22e38", "collapsed": true}, "source": ["def linear_layer(input_, in_size, out_size, name):\n", "    with tf.variable_scope(name):\n", "        W = tf.get_variable('W', initializer=tf.contrib.layers.xavier_initializer(), shape=[out_size, in_size])\n", "        b = tf.get_variable('b', initializer=tf.zeros_initializer(), shape=[out_size])\n", "    return tf.matmul(input_, W, transpose_b=True) + b"], "cell_type": "code", "outputs": [], "execution_count": 6}, {"metadata": {"_uuid": "d9a56c3d5cf32f71c110e1dfa499e79668b4598f", "_cell_guid": "37993dd0-6ac4-41dc-ae30-363bc6f98d4a"}, "cell_type": "markdown", "source": ["### ReLU layer"]}, {"metadata": {"_uuid": "58ffaa4a6cf59ac4b5bf883e5d2612a86cd53648", "_cell_guid": "8d23cfd0-6cfb-4a14-ac33-1f0969210986", "collapsed": true}, "source": ["def relu_layer(input_, in_size, out_size, name):\n", "    return tf.nn.relu(linear_layer(input_, in_size, out_size, name))"], "cell_type": "code", "outputs": [], "execution_count": 7}, {"metadata": {"_uuid": "c6895edb82db1408254d72eb9f32d169a03ed595", "_cell_guid": "13fd7bb9-f535-42dc-b27a-f25e205aeefb"}, "cell_type": "markdown", "source": ["### Forward propagation"]}, {"metadata": {"_uuid": "274c0b3432812f7a851a1226d2dc01c054574f6d", "_cell_guid": "ef622a9f-ff4e-4317-a50a-1f48343db5ce", "collapsed": true}, "source": ["def forward_prop(X, layers):\n", "    for l in range(1, len(layers) - 1):\n", "        with tf.name_scope('relu_layer' + str(l)):\n", "            X = relu_layer(X, layers[l-1], layers[l], 'weights' + str(l))\n", "    l = len(layers) - 1\n", "    with tf.name_scope('linear_layer'):\n", "        X = linear_layer(X, layers[l-1], layers[l], 'weights' + str(l))\n", "    return X"], "cell_type": "code", "outputs": [], "execution_count": 8}, {"metadata": {"_uuid": "1dd2dc137ef75e05e13470721c01dbeb3c8d87b3", "_cell_guid": "b89f78f3-cc34-4d03-8059-dd3fb8e4aba7"}, "cell_type": "markdown", "source": ["### Cost function"]}, {"metadata": {"_uuid": "f5a3466c189bff0be9e22786e1a40f59a61c556c", "_cell_guid": "b5217974-fd5c-45fe-9cff-44228ac080e2", "collapsed": true}, "source": ["def cost_function(logits, labels):\n", "    return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=labels))"], "cell_type": "code", "outputs": [], "execution_count": 9}, {"metadata": {"_uuid": "f84b206a2a1c08188452a85fea03afb1631cbe50", "_cell_guid": "a5e6cb92-f8db-4794-acef-6700ce7b62da"}, "cell_type": "markdown", "source": ["### Predictions and accuracy"]}, {"metadata": {"_uuid": "9c55a2726b2cdba2ea43b52e329dccbe94ff529d", "_cell_guid": "51098602-0248-4359-9a55-6b692b37f0be", "collapsed": true}, "source": ["def logits_to_labels(logits, num_labels):\n", "    return tf.argmax(logits, axis=1)"], "cell_type": "code", "outputs": [], "execution_count": 10}, {"metadata": {"_uuid": "315ff4399265c0f6a9d1c40bc914b5d791a9888f", "_cell_guid": "5754c9e6-b828-400b-8a49-677caddb8b0a", "collapsed": true}, "source": ["def accuracy(predictions, labels):\n", "    correct = tf.equal(predictions, labels)\n", "    acc = tf.reduce_mean(tf.cast(correct, tf.float32))\n", "    return acc"], "cell_type": "code", "outputs": [], "execution_count": 11}, {"metadata": {"_uuid": "d648ea6d37e0c886e868a9dd394371da134c2551", "_cell_guid": "a5110eca-9e27-402f-a123-933d36f23d31"}, "cell_type": "markdown", "source": ["### Putting it all together"]}, {"metadata": {"_uuid": "f85f2a2647698a9542bef878313723ac70b4c07d", "_cell_guid": "0e023dbd-7149-4ed9-8b04-2b3756feea9f", "collapsed": true}, "source": ["def model(X_train, labels_train, X_dev, labels_dev, layers=[784, 200, 10], num_epochs=100, learning_rate=0.01,\n", "          writer_dir='tensorboard/model1/default', checkpoint='checkpoints/model.ckpt'):\n", "    tf.reset_default_graph()\n", "    \n", "    X, labels, Y = input_nodes(layers[0], layers[-1])\n", "    with tf.name_scope('forward_prop'):\n", "        logits = forward_prop(X, layers)\n", "    with tf.name_scope('cost'):\n", "        cost = cost_function(logits, Y)\n", "    with tf.name_scope('accuracy'):\n", "        acc = accuracy(logits_to_labels(logits, layers[-1]), labels)\n", "    with tf.name_scope('summaries'):\n", "        tf.summary.scalar('cost', cost)\n", "        tf.summary.scalar('accuracy', acc)\n", "    \n", "    with tf.name_scope('optimizer'):\n", "        train_step = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n", "    \n", "    with tf.name_scope('summaries'):\n", "        merge = tf.summary.merge_all()\n", "    \n", "    init = tf.global_variables_initializer()\n", "    saver = tf.train.Saver()\n", "    \n", "    graph_writer = tf.summary.FileWriter(writer_dir + '/graph')\n", "    train_writer = tf.summary.FileWriter(writer_dir + '/train')\n", "    dev_writer = tf.summary.FileWriter(writer_dir + '/dev')\n", "    \n", "    with tf.Session() as sess:\n", "        graph_writer.add_graph(sess.graph)\n", "        sess.run(init)\n", "        for i in range(1, num_epochs + 1):\n", "            summary, _, cost_val = sess.run([merge, train_step, cost], { X: X_train, labels: labels_train })\n", "            train_writer.add_summary(summary, i)\n", "            if i % 10 == 0:\n", "                print('{}. iteration: train cost = {}'.format(i, cost_val))\n", "            \n", "            summary, cost_val = sess.run([merge, cost], { X: X_dev, labels: labels_dev })\n", "            dev_writer.add_summary(summary, i)\n", "            if i % 10 == 0:\n", "                print('dev cost = {}'.format(cost_val))\n", "        saver.save(sess, checkpoint)\n", "    \n", "    graph_writer.close()\n", "    train_writer.close()\n", "    dev_writer.close()"], "cell_type": "code", "outputs": [], "execution_count": 12}, {"metadata": {"_uuid": "9d5e8d53bd22eb2cd53ce2e003f6cf37d891506f", "_cell_guid": "950ae410-b4ad-4453-b309-dc0e91253ec3"}, "source": ["model(dataset['train_features'], dataset['train_labels'], dataset['dev_features'], dataset['dev_labels'],\n", "      num_epochs=200, writer_dir='tensorboard/model1/1')"], "cell_type": "code", "outputs": [], "execution_count": 13}, {"metadata": {"_uuid": "962161fcdc83d5af9ae1f37dbc7c9d0a607ad55d", "_cell_guid": "b12b57b3-cae6-4422-9758-aa361d7752af"}, "cell_type": "markdown", "source": ["Houston, we have overfitting! Nevertheless, let's generate our first submission."]}, {"metadata": {"_uuid": "dc4370c074d63c87d1899a2a37b01e95525875f5", "_cell_guid": "d29fb707-9f91-4694-a56b-2aaa3ff3808c", "collapsed": true}, "source": ["def generate_submission(checkpoint, layers):\n", "    tf.reset_default_graph()\n", "    challenge = pd.read_csv('../input/test.csv').values\n", "    X = tf.placeholder(tf.float32, [None, layers[0]])\n", "    logits = forward_prop(X, layers)\n", "    pred = logits_to_labels(logits, layers[-1])\n", "    saver = tf.train.Saver()\n", "    with tf.Session() as sess:\n", "        saver.restore(sess, checkpoint)\n", "        pred = pred.eval({ X: challenge })\n", "    df = pd.DataFrame(data=list(zip(range(1, pred.size+1), pred)), columns=['ImageId', 'Label'])\n", "    return df"], "cell_type": "code", "outputs": [], "execution_count": 16}, {"metadata": {"_uuid": "f0b97dcbb2a76ed7ab2ccd3ca6a79b38d0c0dcb2", "_cell_guid": "e80545e0-7949-44c2-be31-aa21b7f4e591"}, "source": ["df = generate_submission('checkpoints/model.ckpt', [784, 200, 10])\n", "df.to_csv('answer1.csv', index=False, header=True)\n", "df"], "cell_type": "code", "outputs": [], "execution_count": 18}, {"metadata": {"_uuid": "184e3dface2fec9d861602db494cd41be09b82a8", "_cell_guid": "e60ae40d-772c-4547-a0f6-1e80e719fb14", "collapsed": true}, "cell_type": "markdown", "source": ["To be continued..."]}], "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"mimetype": "text/x-python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "name": "python", "version": "3.6.1", "file_extension": ".py", "codemirror_mode": {"version": 3, "name": "ipython"}}}, "nbformat": 4, "nbformat_minor": 1}