{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3c1395e0-8545-1e6f-03bb-e57dc74b8fe8"
      },
      "outputs": [],
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load in \n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "\n",
        "import theano\n",
        "import theano.tensor as T\n",
        "# Input data files are available in the \"../input/\" directory.\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "\n",
        "# Any results you write to the current directory are saved as output."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6b05c6ec-274e-0284-4cd6-225eb1cc0cae"
      },
      "outputs": [],
      "source": [
        "# read training data from CSV file \n",
        "data = pd.read_csv('../input/train.csv')\n",
        "data.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4b9a7083-65ea-e407-d1d4-34ef1ffa0239"
      },
      "outputs": [],
      "source": [
        "# First column is labels, last 784 are flattened 28x28 pixel images\n",
        "images = data.iloc[:,1:].values\n",
        "labels_flat = data.iloc[:,0].values\n",
        "\n",
        "# convert from [0:255] => [0.0:1.0]\n",
        "images = np.multiply(images, 1.0 / 255.0)\n",
        "\n",
        "m = images.shape[0] # Number of training examples\n",
        "n = data.shape[1] # Number of variables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7ec1c9e8-8d18-d93c-258b-23eb5e0c72dd"
      },
      "outputs": [],
      "source": [
        "state = shared(0)\n",
        "inc = T.iscalar('inc')\n",
        "accumulator = function([inc], state, updates=[(state, state+inc)])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3412c330-0148-28a9-7cc7-fa74b916ae99"
      },
      "outputs": [],
      "source": [
        "# display image\n",
        "def display(img):\n",
        "    \n",
        "    # (784) => (28,28)\n",
        "    one_image = img.reshape(28, 28)\n",
        "    plt.imshow(one_image, cmap='gray')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d545a8b9-ddd1-0b5b-9c84-692e2daae37b"
      },
      "outputs": [],
      "source": [
        "# output image\n",
        "imageToDisplay = 999\n",
        "display(images[imageToDisplay])\n",
        "labels_flat[imageToDisplay]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "24fbfae4-8912-7737-ed37-466f5df22e41"
      },
      "outputs": [],
      "source": [
        "# There are 10 unique class labels we need to predict\n",
        "labels_count = np.unique(labels_flat).shape[0]\n",
        "\n",
        "print('labels_count => {0}'.format(labels_count))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d298987-0b6e-bb82-fc93-c9124f193c58"
      },
      "outputs": [],
      "source": [
        "# convert class labels from scalars to one-hot vectors\n",
        "# 0 => [1 0 0 0 0 0 0 0 0 0]\n",
        "# 1 => [0 1 0 0 0 0 0 0 0 0]\n",
        "# ...\n",
        "# 9 => [0 0 0 0 0 0 0 0 0 1]\n",
        "\n",
        "def dense_to_one_hot(labels_dense, num_classes):\n",
        "    num_labels = labels_dense.shape[0]\n",
        "    index_offset = np.arange(num_labels) * num_classes\n",
        "    labels_one_hot = np.zeros((num_labels, num_classes))\n",
        "    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1\n",
        "    return labels_one_hot\n",
        "\n",
        "labels = dense_to_one_hot(labels_flat, labels_count)\n",
        "labels = labels.astype(np.uint8)\n",
        "\n",
        "print('labels({0[0]},{0[1]})'.format(labels.shape))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c1d40094-02df-221e-0d6a-a59248e72b83"
      },
      "outputs": [],
      "source": [
        "print(labels[48])\n",
        "print(labels_flat[48])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b02488d5-6fcd-d2a8-a218-12b1b4e256bb"
      },
      "outputs": [],
      "source": [
        "w = np.zeros((784,10))\n",
        "print(w.shape)\n",
        "print(images.shape)\n",
        "print(np.dot(images, w).shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "438e13e2-3f50-5de9-2809-5f0ac1e6c415"
      },
      "outputs": [],
      "source": [
        "np.argmax([0,0,0,1,0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "229a3c26-b842-4a1e-eaab-0aa5d38ebcbb"
      },
      "outputs": [],
      "source": [
        "# Machine learning parameters\n",
        "lam = 0.01 # Regularization parameter\n",
        "alpha = 0.1 # Gradient descent step function\n",
        "training_steps = 10000 # Number of training steps"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7bf13ee6-e95d-1871-ca98-5e9598fa252f"
      },
      "outputs": [],
      "source": [
        "# Declare Theano symbolic variables\n",
        "x = T.dmatrix(\"x\")\n",
        "y = T.dvector(\"y\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ada06a49-8f3a-f183-3369-c7886bf26c7e"
      },
      "outputs": [],
      "source": [
        "# w is shared so it can keep its values between training iterations\n",
        "w = theano.shared(np.random.rand(784, 10), name=\"w\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "74888a18-0025-64b7-4add-6d6dbfbd466a"
      },
      "outputs": [],
      "source": [
        "# Construct Theano expression graph\n",
        "p_1 = 1 / (1 + T.exp(-T.dot(x,w))) # Probability that target = 1\n",
        "prediction = p_1.argmax(axis = 1) # The prediction threshold\n",
        "xent = -y * T.log(p_1) - (1 - y) * T.log(1-p_1) # Cross-entropy loss function\n",
        "cost = xent.mean(axis = 0)\n",
        "gw0 = T.grad(cost[0], w) "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "91949fd1-13f0-7601-c590-45fa57d7a1a2"
      },
      "outputs": [],
      "source": [
        "# Compile\n",
        "train = theano.function(\n",
        "            inputs = [x,y],\n",
        "            outputs = [prediction, cost],\n",
        "            updates = [(w, w - alpha * gw0)],\n",
        "            )\n",
        "predict = theano.function(inputs=[x], outputs=prediction)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "e578ba63-3b0c-9098-bd18-9ea3f0b45354"
      },
      "outputs": [],
      "source": [
        "# Train\n",
        "for i in range(training_steps):\n",
        "    pred, err = train(images, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "693b5b8d-d95a-bbba-8c31-6cdded74a8de"
      },
      "outputs": [],
      "source": [
        "predict(images)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c5ce1204-5be5-7d6f-b935-39db80799e1a"
      },
      "outputs": [],
      "source": [
        "state = theano.shared(0)\n",
        "inc = T.iscalar('inc')\n",
        "accumulator = theano.function([inc], state, updates=[(state, state+inc)])\n",
        "accumulator(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9b5e2c11-f26b-7189-1a7b-e71fc1eee62d"
      },
      "outputs": [],
      "source": [
        "accumulator(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "43bbd38b-00d4-d744-8fad-c27a19f1fe63"
      },
      "outputs": [],
      "source": [
        "accumulator(2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "f23cb672-04e7-85ea-0c65-641b37d7dd71"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}