{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6b54d554-2dec-2f59-01ab-eeb6978f63db"
      },
      "source": [
        "Welcome to this example of (sort of deep) convolutional neural nets with keras. It uses the Tensorflow backend and makes use of the following:\n",
        "\n",
        " - Dropout\n",
        " - Maximum weight constraints (suggested in dropout paper)\n",
        " - Convolutional Layers\n",
        " - Callbacks that change parameters"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6aebfc71-6d6d-de15-b7aa-171f61999b0e"
      },
      "outputs": [],
      "source": [
        "#%env KERAS_BACKEND=theano\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n",
        "print(check_output([\"free\", \"-mh\"]).decode(\"utf8\"))\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        " \n",
        "import keras\n",
        "import keras.backend as K\n",
        "from keras.layers import Input, Convolution2D, Activation, MaxPooling2D, \\\n",
        "     Dense, BatchNormalization, Dropout\n",
        "from keras.layers.core import Flatten\n",
        "from keras.optimizers import SGD\n",
        "from keras.models import Model\n",
        "from keras.utils import np_utils\n",
        "from keras.constraints import maxnorm\n",
        "from keras.regularizers import l2\n",
        "from keras.callbacks import LearningRateScheduler\n",
        "\n",
        "print(keras.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "9195fa17-16ae-196c-90c9-b1823eb18715"
      },
      "outputs": [],
      "source": [
        "train = np.genfromtxt('../input/train.csv', delimiter = ',', skip_header = 1)\n",
        "training_inputs = train[:, 1:] / 127.5 - 1\n",
        "\n",
        "training_targets = np_utils.to_categorical(train[:, int(0)])\n",
        "\n",
        "test = np.genfromtxt('../input/test.csv', delimiter = ',', skip_header = 1)\n",
        "test_inputs = test[:, ] / 127.5 - 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "086037ea-421f-31e0-df3d-7faa15e4255b"
      },
      "outputs": [],
      "source": [
        "# For 2D data (e.g. image), ordering type \"tf\" assumes (rows, cols, channels)\n",
        "#  type \"th\" assumes (channels, rows, cols). See https://keras.io/backend/\n",
        "print('We are using image ordering type', K.image_dim_ordering())\n",
        "\n",
        "training_inputs = training_inputs.reshape(training_inputs.shape[0], 28, 28, 1)\n",
        "test_inputs = test_inputs.reshape(test_inputs.shape[0], 28, 28, 1)\n",
        "print(training_inputs.shape)\n",
        "print(test_inputs.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3be466c5-fe04-5548-fff8-e29741b5a883"
      },
      "outputs": [],
      "source": [
        "inputs = Input(shape = (28, 28, 1))\n",
        "print(inputs._keras_shape)\n",
        "\n",
        "inputs_w_dropout = Dropout(.10)(inputs)\n",
        "print(inputs._keras_shape)\n",
        "\n",
        "conv1 = Convolution2D(30, 5, 5, #dim_ordering = 'th',\n",
        "                      border_mode = 'valid', bias = True,\n",
        "                      W_constraint = maxnorm(2.5))(inputs_w_dropout)\n",
        "print(conv1._keras_shape)\n",
        "\n",
        "max1 = MaxPooling2D(pool_size = (2, 2))(conv1)\n",
        "max1 = Dropout(.05)(max1)\n",
        "max1 = BatchNormalization()(max1)\n",
        "act1 = Activation('relu')(max1)\n",
        "print(act1._keras_shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1ec59a2a-8774-b1aa-aeba-3dff53c26b1b"
      },
      "outputs": [],
      "source": [
        "conv2 = Convolution2D(45, 4, 4, #dim_ordering = 'th',\n",
        "                      border_mode = 'valid',\n",
        "                      bias = True, W_constraint = maxnorm(2.5))(act1)\n",
        "print('Conv2 shape: ', conv2._keras_shape)\n",
        "max2 = MaxPooling2D(pool_size = (2, 2))(conv2)\n",
        "max2 = Dropout(.05)(max2)\n",
        "max2 = BatchNormalization()(max2)\n",
        "act2 = Activation('relu')(max2)\n",
        "print('act2 shape: ', act2._keras_shape)\n",
        "\n",
        "conv3 = Convolution2D(160, 2, 2, #dim_ordering = 'th',\n",
        "                      border_mode = 'valid',\n",
        "                      bias = True, W_constraint = maxnorm(2.5))(act2)\n",
        "print(conv3._keras_shape)\n",
        "conv3 = Dropout(.05)(conv3)\n",
        "conv3 = BatchNormalization()(conv3)\n",
        "act3 = Activation('relu')(conv3)\n",
        "print(act3._keras_shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "571b8a93-c003-7544-d232-84583568d7cb"
      },
      "outputs": [],
      "source": [
        "flat1 = Flatten()(act3)\n",
        "print(flat1._keras_shape)\n",
        "\n",
        "dense1 = Dense(800, W_constraint = maxnorm(2))(flat1)\n",
        "dense1 = Dropout(.5)(dense1)\n",
        "dense1 = BatchNormalization()(dense1)\n",
        "dense1 = Activation('softmax')(dense1)\n",
        "print(dense1._keras_shape)\n",
        "\n",
        "dense2 = Dense(10)(dense1)\n",
        "out = Activation('softmax')(dense2)\n",
        "print(dense2._keras_shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "48bf1e77-dc85-1517-0b1f-e98865b8654a"
      },
      "outputs": [],
      "source": [
        "sgd = SGD(lr = 0.5, decay = 0, momentum = .9, nesterov = True)\n",
        " \n",
        "model = Model(input = inputs, output = out)\n",
        "model.compile(loss='categorical_crossentropy', optimizer = sgd,\n",
        "              metrics = ['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a2a2eb6c-15ac-dbb2-ed75-74379303142b"
      },
      "outputs": [],
      "source": [
        "# I can't seem to change the model fit parameters!\n",
        "class MyCallback(keras.callbacks.Callback):\n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        print(\"Callback initiated: on_epoch_begin\")\n",
        "        self.params['batch_size'] = 1000\n",
        "        \n",
        "my_callback = MyCallback()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "5a4bab13-b766-5aca-cea5-bfedce9071c0"
      },
      "outputs": [],
      "source": [
        "# Creating a callback function to alter the learning rate and momentum in custom fashion\n",
        "# TODO: Investigate ReduceLROnPlateau at https://keras.io/callbacks/\n",
        "def scheduler(epoch):\n",
        "    print(\"I am epoch \" + str(epoch))\n",
        "    if epoch == 1:\n",
        "        sgd.lr = K.variable(.01)\n",
        "        sgd.momentum = K.variable(.94)\n",
        "    if epoch == 4:\n",
        "        sgd.lr = K.variable(.001)\n",
        "        sgd.momentum = K.variable(.96)\n",
        "    if epoch == 10:\n",
        "        sgd.lr = K.variable(.0001)\n",
        "        sgd.momentum = K.variable(.8)\n",
        "    if epoch == 18:\n",
        "        sgd.lr = K.variable(.0001)\n",
        "        sgd.momentum = K.variable(0)\n",
        "    current_lr = model.optimizer.get_config()['lr']\n",
        "    current_momentum = model.optimizer.get_config()['momentum']\n",
        "    print(\"My lr is \" + str(current_lr))\n",
        "    print(\"My momentum is \" + str(current_momentum))   \n",
        "    return float(current_lr)\n",
        "\n",
        "change_lr = LearningRateScheduler(scheduler)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "89b46280-d9a2-890c-64a0-9d3f668c1bb9"
      },
      "outputs": [],
      "source": [
        "fit = model.fit(training_inputs, training_targets, nb_epoch = 10,\n",
        "                batch_size = 800, validation_split = 0.15,\n",
        "                shuffle = True,\n",
        "                callbacks = [change_lr, my_callback])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "902c2a48-ae19-e897-7ad1-a8dea6c8ccc8"
      },
      "outputs": [],
      "source": [
        "preds = model.predict(test_inputs)\n",
        "pred_classes = preds.argmax(axis = -1)\n",
        "content = {\"ImageId\": range(1, len(preds) + 1), \"Label\": pred_classes}\n",
        "pd.DataFrame(content).to_csv(\"kernel_pred.csv\", index = False)\n",
        "print(check_output([\"ls\", \".\"]).decode(\"utf8\"))"
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}