{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cc1588fa-89e5-f4a4-5b7d-5dfc3ee463c8"
      },
      "outputs": [],
      "source": [
        "from __future__ import print_function\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from six.moves import cPickle as pickle\n",
        "from six.moves import range\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "from keras.utils.np_utils import to_categorical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a1df0f87-7595-286d-2308-b2f92a5c49a1"
      },
      "outputs": [],
      "source": [
        "train_df = pd.read_csv(\"../input/train.csv\")\n",
        "test_dataset = pd.read_csv(\"../input/test.csv\")\n",
        "train_dataset = train_df.drop(['label'], axis=1).values.astype('float32')\n",
        "train_labels = train_df['label'].values\n",
        "test_dataset = test_dataset.values.astype('float32')\n",
        "print(train_dataset.shape)\n",
        "print(train_labels.shape)\n",
        "#print(test_dataset.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "70662aa1-858d-b468-7bcd-7bd481d37924"
      },
      "outputs": [],
      "source": [
        "image_size = 28\n",
        "num_labels = 10\n",
        "num_channels = 1 # grayscale\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "def reformat(dataset):\n",
        "  dataset = dataset.reshape(\n",
        "    (-1, image_size, image_size, num_channels)).astype(np.float32)\n",
        "  return dataset\n",
        "train_dataset = reformat(train_dataset)\n",
        "#valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
        "test_dataset = reformat(test_dataset)\n",
        "train_labels = to_categorical(train_labels)\n",
        "print('Training set', train_dataset.shape)\n",
        "print (train_labels.shape)\n",
        "#print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
        "print('Test set', test_dataset.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f0e27efc-8d87-7c3c-ed95-c32a09904ad9"
      },
      "outputs": [],
      "source": [
        "def accuracy(predictions, labels):\n",
        "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
        "          / predictions.shape[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f0c851d9-1c2c-eadd-2981-a07fedb82191"
      },
      "outputs": [],
      "source": [
        "batch_size = 16\n",
        "patch_size = 3\n",
        "depth = 16\n",
        "num_hidden = 64\n",
        "graph = tf.Graph()\n",
        "with graph.as_default():\n",
        "    model_train_data = tf.placeholder(tf.float32,shape=[batch_size,image_size,image_size,num_channels])\n",
        "    model_train_labels = tf.placeholder(tf.float32,shape=[batch_size,num_labels])\n",
        "    #model_validation_data = tf.constant(valid_dataset)\n",
        "    model_test_data = tf.constant(test_dataset)\n",
        "    test_image = tf.placeholder(tf.float32,shape=[1,image_size,image_size,num_channels])\n",
        "    #layer matrices\n",
        "    #conv layer1\n",
        "    layer1_weights = tf.Variable(tf.truncated_normal([patch_size,patch_size,num_channels,depth],stddev=0.1),name='layer1_w')\n",
        "    layer1_bias = tf.Variable(tf.zeros([depth]),name='layer1_b')\n",
        "    #conv layer2\n",
        "    layer2_weights = tf.Variable(tf.truncated_normal([patch_size,patch_size,depth,depth],stddev=0.1),name='layer2_w')\n",
        "    layer2_bias = tf.Variable(tf.constant(1.0,shape=[depth]),name='layer2_b')\n",
        "    \n",
        "    #fully hidden1\n",
        "    shape = ((image_size-patch_size+1)//2 - patch_size +1)//2\n",
        "    layer3_weights = tf.Variable(tf.truncated_normal([shape*shape*depth,num_hidden],stddev=0.1),name='layer3_w')\n",
        "    layer3_bias = tf.Variable(tf.constant(1.0,shape=[num_hidden]),name='layer3_b')\n",
        "    #output\n",
        "    layer4_weights = tf.Variable(tf.truncated_normal([num_hidden,num_labels],stddev=0.1),name='layer4_w')\n",
        "    layer4_bias = tf.Variable(tf.constant(1.0,shape=[num_labels]),name='layer4_b')\n",
        "    \n",
        "    def model(data):\n",
        "        with tf.name_scope('layer1'):\n",
        "            with tf.name_scope('weights'):\n",
        "                conv1 = tf.nn.conv2d(data,layer1_weights,[1, 1, 1, 1],padding='VALID')\n",
        "            with tf.name_scope('biases'):\n",
        "                bias1 = tf.nn.relu(conv1+layer1_bias)\n",
        "\n",
        "        #pooling 1\n",
        "        with tf.name_scope('pool1'):\n",
        "            pool1 = tf.nn.avg_pool(bias1,[1, 2, 2, 1],[1, 2, 2, 1],padding='VALID')\n",
        "        with tf.name_scope('layer2'):\n",
        "            with tf.name_scope('weights'):\n",
        "                conv2 = tf.nn.conv2d(pool1,layer2_weights,[1, 1, 1, 1],padding='VALID')\n",
        "            with tf.name_scope('biases'):\n",
        "                bias2 = tf.nn.relu(conv2 + layer2_bias)\n",
        "        #pooing 2\n",
        "        \n",
        "        with tf.name_scope('pool2'):\n",
        "            pool2 = tf.nn.avg_pool(bias2,[1, 2, 2, 1],[1, 2, 2, 1],padding='VALID')\n",
        "        \n",
        "        shape = pool2.get_shape().as_list()\n",
        "        reshape = tf.reshape(pool2, [shape[0], shape[1] * shape[2] * shape[3]])\n",
        "        with tf.name_scope('layer3'):\n",
        "            hidden = tf.nn.relu(tf.matmul(reshape, layer3_weights) + layer3_bias)\n",
        "        return tf.matmul(hidden, layer4_weights) + layer4_bias\n",
        "    with tf.name_scope('logits'):\n",
        "        logit = model(model_train_data)\n",
        "    with tf.name_scope('loss'):\n",
        "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logit,labels=model_train_labels))\n",
        "    #optmizer\n",
        "    with tf.name_scope('train'):\n",
        "        optimizer = tf.train.AdamOptimizer(0.03).minimize(loss)\n",
        "    #tf.histogram_summary(\"weights1\", layer1_weights)\n",
        "    #tf.histogram_summary(\"biases1\", layer1_bias)\n",
        "    #tf.histogram_summary(\"weights2\", layer2_weights)\n",
        "    #tf.histogram_summary(\"biases2\", layer2_bias)\n",
        "    #tf.histogram_summary(\"weights3\", layer3_weights)\n",
        "    #tf.histogram_summary(\"biases3\", layer3_bias)\n",
        "    #tf.histogram_summary(\"weights4\", layer4_weights)\n",
        "    #tf.histogram_summary(\"biases4\", layer4_bias)\n",
        "    #tf.scalar_summary('loss',loss)\n",
        "    #summarizer = tf.merge_all_summaries()\n",
        "    train_prediction = tf.nn.softmax(logit)\n",
        "    #valid_prediction = tf.nn.softmax(model(model_validation_data))\n",
        "    test_prediction = tf.nn.softmax(model(model_test_data))\n",
        "    image_prediction = tf.nn.softmax(model(test_image))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4abc1d73-ffed-d1ca-3bc3-4d4f5449bc78"
      },
      "outputs": [],
      "source": [
        "num_steps = 1201\n",
        "\n",
        "with tf.Session(graph=graph) as session:\n",
        "    tf.initialize_all_variables().run()\n",
        "    print('Initialized')\n",
        "    saver = tf.train.Saver()\n",
        "    for step in range(num_steps):\n",
        "        offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
        "        batch_data = train_dataset[offset:(offset + batch_size), :, :, :]\n",
        "        batch_labels = train_labels[offset:(offset + batch_size), :]\n",
        "        feed_dict = {model_train_data : batch_data, model_train_labels : batch_labels}\n",
        "        _, l, predictions= session.run(\n",
        "          [optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
        "        if (step % 200 == 0):\n",
        "            print('Minibatch loss at step %d: %f' % (step, l))\n",
        "            print('Minibatch accuracy: %.1f%%' % accuracy(predictions, batch_labels))\n",
        "            #train_writer.add_summary(summary, step)\n",
        "#     saver.save(session, \"./model.ckpt\")\n",
        "    np.savetxt('mnist_output.csv', np.c_[range(1,len(test_dataset)+1),np.argmax(test_prediction.eval(), 1)], delimiter=',', header = 'ImageId,Label', comments = '', fmt='%d')"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}