{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "3116c2b4-b378-2727-c6b6-ba31ba9d8c11"
      },
      "source": [
        "https://www.tensorflow.org/get_started/mnist/pros#f1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b511e09b-7b7c-9df2-6247-894a8362a0b3"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm, tqdm_notebook\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aa8fbdb4-c838-354f-83aa-9e6bc0581f50"
      },
      "outputs": [],
      "source": [
        "train = pd.read_csv('../input/train.csv')\n",
        "test = pd.read_csv('../input/test.csv')\n",
        "print(train.shape)\n",
        "print(test.shape)\n",
        "\n",
        "X = train.drop('label', axis = 1).values\n",
        "y = train['label'].values\n",
        "x_test_to_submit = test.values\n",
        "del train, test\n",
        "print(x.shape, y.shape, x_test_to_submit.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "840bb7b5-a2ad-2a22-4206-631fc61c5d38"
      },
      "outputs": [],
      "source": [
        "#function to One-Hot-Encoding target variable\n",
        "#example: 3 ---> [0,0,0,1,0,0,0,0,0,0]\n",
        "def encodeTarget(Y):\n",
        "    temp = np.zeros((y.size, y.max()+1))\n",
        "    temp[np.arange(y.size), y] = 1\n",
        "    return temp\n",
        "\n",
        "y = encodeTarget(y)\n",
        "\n",
        "#function for digits visualization  \n",
        "def printDigit(matrix, hot_label):\n",
        "    matrix = matrix.reshape((28, 28))\n",
        "    label = np.where(hot_label==1)[0][0]  #decode hot-encoded label\n",
        "    plt.figure(figsize = (1.5,1.5))\n",
        "    plt.title('Label is {label}'.format(label=label))\n",
        "    plt.imshow(matrix, cmap='plasma')\n",
        "    plt.show()\n",
        "    \n",
        "printDigit(X[7,:], y[7])\n",
        "printDigit(X[777,:], y[777])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b650355c-d2ab-8dde-8e9f-a8713eb734e1"
      },
      "outputs": [],
      "source": [
        "sess = tf.InteractiveSession()\n",
        "x = tf.placeholder(tf.float32, shape=[None, 784])  #features\n",
        "y_ = tf.placeholder(tf.float32, shape=[None, 10])  #true targets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4f4ea3ef-3190-d9a5-b694-8af648abb925"
      },
      "outputs": [],
      "source": [
        "# weight inotialization\n",
        "def weight_variable(shape):\n",
        "    initial = tf.truncated_normal(shape, mean=0.0, stddev=0.1)   #init by random values ~N(0.0,0.1)\n",
        "    return tf.Variable(initial)\n",
        "\n",
        "def bias_variable(shape):\n",
        "    initial = tf.constant(0.1, shape=shape)   #init by constant\n",
        "    return tf.Variable(initial)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8152b058-94b5-2a73-e0ca-c78f79783723"
      },
      "outputs": [],
      "source": [
        "# convolution and pooling\n",
        "\n",
        "def conv2d(x, W):\n",
        "    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
        "\n",
        "def max_pool_2x2(x):\n",
        "    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
        "                        strides=[1, 2, 2, 1], padding='SAME')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "06150709-04fe-4c61-fdc9-8ead468b35e1"
      },
      "outputs": [],
      "source": [
        "# First Convolutional Layer\n",
        "W_conv1 = weight_variable([5, 5, 1, 32])     #32 maps, tune\n",
        "b_conv1 = bias_variable([32])\n",
        "\n",
        "x_image = tf.reshape(x, [-1,28,28,1])\n",
        "#-1 - is the special value, the size of that dimension is computed so that the total size remains constant\n",
        "#28,28 - width and height\n",
        "#1 - RGB numbers (we have black and white pics, therefore we have only 1 number)\n",
        "\n",
        "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
        "h_pool1 = max_pool_2x2(h_conv1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b0140e85-4766-ffca-ada9-395f6fdd1e88"
      },
      "outputs": [],
      "source": [
        "# second convolutional layer\n",
        "\n",
        "W_conv2 = weight_variable([5, 5, 32, 64])    #64 maps, tune\n",
        "b_conv2 = bias_variable([64])\n",
        "\n",
        "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
        "h_pool2 = max_pool_2x2(h_conv2)  #28*28 --> 14*14"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8c8aa50d-59a6-2cd0-2a9c-f6f1a7975ab7"
      },
      "outputs": [],
      "source": [
        "# densely connected layer\n",
        " \n",
        "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
        "b_fc1 = bias_variable([1024])\n",
        "\n",
        "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
        "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f7801d20-0d9d-0062-c3e4-60cd838f57b0"
      },
      "outputs": [],
      "source": [
        "# dropout\n",
        "keep_prob = tf.placeholder(tf.float32)\n",
        "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b2192a4c-6e37-6b79-b997-0f7c9b48398f"
      },
      "outputs": [],
      "source": [
        "# readout layer\n",
        "W_fc2 = weight_variable([1024, 10])\n",
        "b_fc2 = bias_variable([10])\n",
        "\n",
        "y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e9fc8729-ab5d-d801-d300-5628f83c3676"
      },
      "outputs": [],
      "source": [
        "#get next batch\n",
        "def next_batch(batch_size):\n",
        "    global data_index\n",
        "    batch = X[data_index:data_index+batch_size,:]\n",
        "    labels = y[data_index:data_index+batch_size,:]\n",
        "    return (batch, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c037b31c-148d-09ec-0226-109d9d1f956d"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "# Train the Model\n",
        "\n",
        "data_index = 0\n",
        "batch_size = 25\n",
        "\n",
        "cross_entropy = tf.reduce_mean(\n",
        "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n",
        "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
        "correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
        "sess.run(tf.global_variables_initializer())\n",
        "for i in range(1000):\n",
        "    batch = next_batch(batch_size)\n",
        "    if i%100 == 0:\n",
        "        train_accuracy = accuracy.eval(feed_dict={\n",
        "            x:batch[0], y_: batch[1], keep_prob: 1.0})\n",
        "        print(\"step %d, training accuracy %g\"%(i, train_accuracy))\n",
        "    train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})\n",
        "\n",
        "#print(\"test accuracy %g\"%accuracy.eval(feed_dict={\n",
        "#    x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "49fd279f-ec68-500e-783a-630514433cd2"
      },
      "outputs": [],
      "source": [
        "#predict by batch (without such tric Memory Error raises):\n",
        "size = 100\n",
        "batches_generator = (x_test_to_submit[i:i + size] for i in range(0, len(x_test_to_submit), size))\n",
        "predictions = []\n",
        "y_pred_digits = tf.argmax(y_conv,1)\n",
        "for test_batch in batches_generator:\n",
        "    predictions.extend(y_pred_digits.eval(feed_dict={x:test_batch, keep_prob: 1.0}))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cc8053c3-5c7a-d6af-5d83-9b303beec1ed"
      },
      "outputs": [],
      "source": [
        "#save results\n",
        "header = \"ImageId,Label\\n\"\n",
        "with open(\"results.csv\", 'w') as f:\n",
        "    f.write(header)\n",
        "    f.write(\"\\n\".join((\"{},{}\".format(ind+1, dig) for ind,dig in enumerate(predictions))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ea9d77b9-3432-47a5-b42d-9671cbaf634f"
      },
      "outputs": [],
      "source": [
        "ls"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "753d5133-3f66-ecff-37af-4a2b0d5c35f0"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}