{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "7ab2d32c-fac1-83fb-fae5-f8fe8a939b38"
      },
      "source": [
        "This is my first Notebook on Kaggle.\n",
        "What I actually did is to adapt the MNIST tutorial on tensorflow website to use the input of Kaggle. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2644f3bf-214c-3911-0d54-8dc8dd87c8be"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import random \n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.cm as cm\n",
        "\n",
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ede7c4ee-cbea-d7fd-b6a6-2bea6f04eb0a"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('../input/train.csv') \n",
        "data.reindex(np.random.permutation(data.index))\n",
        "trainImages = data.drop('label', 1)\n",
        "trainLabels = data['label']\n",
        "\n",
        "testImages = pd.read_csv('../input/test.csv')\n",
        "\n",
        "print('Data imported seccessfully ')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c1d0aca5-9e21-8595-1eb6-4f85a4dd8e2e"
      },
      "outputs": [],
      "source": [
        "expInd = 724\n",
        "plt.imshow(trainImages.iloc[expInd].values.reshape(28,28), cmap='Greys_r')\n",
        "print(trainLabels.iloc[expInd].values.argmax())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "11617be2-b060-208d-8730-39de738bf537"
      },
      "outputs": [],
      "source": [
        "# Create a TF interactive session \n",
        "sess = tf.InteractiveSession()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f38d1bb5-3e04-9d25-d90d-0bb71cfe2005"
      },
      "outputs": [],
      "source": [
        "# Weight Initialization \n",
        "def weight_variable(shape):\n",
        "\tinitial = tf.truncated_normal(shape, stddev=0.1)\n",
        "\treturn tf.Variable(initial)\n",
        "\n",
        "def bias_variable(shape):\n",
        "\tinitial = tf.constant(0.1, shape=shape)\n",
        "\treturn tf.Variable(initial)\n",
        "\n",
        "# Convolution and Pooling \n",
        "def conv2d(x, W): \n",
        "\treturn tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
        "\n",
        "def max_pool_2x2(x):\n",
        "\treturn tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5edc0d79-1731-fe64-189a-148554e29a26"
      },
      "outputs": [],
      "source": [
        "# Create the model\n",
        "x = tf.placeholder(tf.float32, [None, 784])\n",
        "x_image = tf.reshape(x, [-1,28,28,1])\n",
        "\n",
        "y_ = tf.placeholder(tf.float32, [None, 10])\n",
        "\n",
        "# First Convolutional Layer \n",
        "W_conv1 = weight_variable([5, 5, 1, 32])\n",
        "b_conv1 = bias_variable([32])\n",
        "\n",
        "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
        "h_pool1 = max_pool_2x2(h_conv1)\n",
        "\n",
        "# Second Convolutional Layer \n",
        "W_conv2 = weight_variable([5, 5, 32, 64])\n",
        "b_conv2 = bias_variable([64])\n",
        "\n",
        "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
        "h_pool2 = max_pool_2x2(h_conv2)\n",
        "\n",
        "# Densely Connected Layer \n",
        "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
        "b_fc1 = bias_variable([1024])\n",
        "\n",
        "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
        "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
        "\n",
        "# Dropout \n",
        "keep_prob = tf.placeholder(tf.float32)\n",
        "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
        "\n",
        "# Readout Layer \n",
        "W_fc2 = weight_variable([1024, 10])\n",
        "b_fc2 = bias_variable([10])\n",
        "\n",
        "y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ef31d1df-c8ba-9e28-91eb-1bf976ab7555"
      },
      "outputs": [],
      "source": [
        "digits = np.matrix([[0,1,2,3,4,5,6,7,8,9],]*42000)\n",
        "\n",
        "tl = np.matrix(trainLabels.values).transpose()\n",
        "trainLabels = (tl == digits).astype(float)\n",
        "trainLabels = pd.DataFrame(trainLabels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3d7082af-f712-736f-322b-5711887d8cf8"
      },
      "outputs": [],
      "source": [
        "# Train and Evaluate the Model \n",
        "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.clip_by_value(y_conv,1e-10,1.0)), reduction_indices=[1]))\n",
        "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
        "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "\n",
        "sess.run(tf.initialize_all_variables())\n",
        "for i in range(400):\n",
        "    iStart = i*100\n",
        "    iEnd = (i+1)*100\n",
        "    batch_xs, batch_ys = trainImages.iloc[iStart:iEnd].values, trainLabels.iloc[iStart:iEnd].values\n",
        "    train_accuracy = accuracy.eval(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 1.0}, session=sess)\n",
        "    print(\"step %d, training accuracy %g\"%(iStart, train_accuracy))\n",
        "    train_step.run(feed_dict={x: batch_xs, y_: batch_ys, keep_prob: 0.5}, session=sess)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "91488ac2-40f4-3833-8904-1f858368c2f8"
      },
      "outputs": [],
      "source": [
        "# Test trained model\n",
        "correct_prediction = y_conv\n",
        "\n",
        "predict = correct_prediction.eval(feed_dict={x: testImages.iloc[0:100].values, keep_prob: 1.0}, session=sess)\n",
        "\n",
        "for ii in range(279): \n",
        "    i = ii + 1\n",
        "    iStart = i*100\n",
        "    iEnd = (i+1)*100\n",
        "    predict = np.append(predict, correct_prediction.eval(feed_dict={x: testImages.iloc[iStart:iEnd].values, keep_prob: 1.0}, session=sess), axis=0)\n",
        "    print(\"Prediction shape: \", predict.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7d7144b0-a81e-0370-14f8-eabee745435c"
      },
      "outputs": [],
      "source": [
        "predict.argmax(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "166fdee2-df0a-0609-ed71-ab10e0f71d53"
      },
      "outputs": [],
      "source": [
        "pred = pd.DataFrame(data = predict, columns = {'Label'}, dtype=int)\n",
        "pred.index += 1 \n",
        "pred.index.name = 'ImageId' \n",
        "pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6b9fefec-ab4a-466f-a431-e89299e321a9"
      },
      "outputs": [],
      "source": [
        "print(testImages.iloc[0].values.shape)pred.to_csv(path_or_buf='submission.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1163f9e3-0fa2-7b7d-08f7-add009fe8c67"
      },
      "outputs": [],
      "source": [
        "plt.imshow(testImages.iloc[3].values.reshape(28,28), cmap='Greys_r')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7df58bdb-cf07-eddf-3813-d7991b1eb4a7"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}