{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b8f23803-f7c9-c890-22dd-eab6672e749c"
      },
      "source": [
        "My first ever Keras notebook, and what better place to start than MNIST?"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "19d8eb6b-9b71-029c-06e1-7e74975ae669"
      },
      "outputs": [],
      "source": [
        "import numpy as np # linear algebra\n",
        "\n",
        "from subprocess import check_output\n",
        "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "76adcf6f-d109-e47d-d8f0-c7a26e986df5"
      },
      "outputs": [],
      "source": [
        "from keras.utils import np_utils\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten, Activation, Conv2D, MaxPooling2D\n",
        "from keras.optimizers import Adam\n",
        "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "from keras.preprocessing.image import ImageDataGenerator"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "249daad0-7a0d-ad43-2bf9-4861ffac0d90"
      },
      "outputs": [],
      "source": [
        "train_file = \"../input/train.csv\"\n",
        "test_file = \"../input/test.csv\"\n",
        "output_file = \"submission.csv\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b8f5eb0f-6bd4-5484-0a89-d7f814678c80"
      },
      "outputs": [],
      "source": [
        "mnist_dataset = np.loadtxt(train_file, skiprows=1, dtype='int', delimiter=',')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1d2ceaed-52a4-581b-51af-c139e4913ee4"
      },
      "outputs": [],
      "source": [
        "val_split = 0.125\n",
        "n_raw = mnist_dataset.shape[0]\n",
        "n_val = int(n_raw * val_split + 0.5)\n",
        "n_train = n_raw - n_val\n",
        "\n",
        "np.random.shuffle(mnist_dataset)\n",
        "x_val, x_train = mnist_dataset[:n_val,1:], mnist_dataset[n_val:,1:]\n",
        "y_val, y_train = mnist_dataset[:n_val,0], mnist_dataset[n_val:,0]\n",
        "\n",
        "x_train = x_train.astype(\"float32\")/255.0\n",
        "x_val = x_val.astype(\"float32\")/255.0\n",
        "y_train = np_utils.to_categorical(y_train)\n",
        "y_val = np_utils.to_categorical(y_val)\n",
        "\n",
        "n_classes = y_train.shape[1]\n",
        "x_train = x_train.reshape(n_train, 28, 28, 1)\n",
        "x_val = x_val.reshape(n_val, 28, 28, 1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "82eb74e2-1f8d-02e9-cf9b-cbdd6c7d371d"
      },
      "source": [
        "I copied this net from several examples I found online. Feel free to modify the layers."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "363e8bc8-a731-01da-9a39-d3914ea7a5c8"
      },
      "outputs": [],
      "source": [
        "model = Sequential()\n",
        "\n",
        "model.add(Conv2D(filters = 32, kernel_size = (3, 3), input_shape = (28, 28, 1), padding='same'))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Conv2D(filters = 32, kernel_size = (3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Dropout(0.25))\n",
        "\n",
        "model.add(Conv2D(filters = 64, kernel_size = (3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Conv2D(filters = 64, kernel_size = (3, 3)))\n",
        "model.add(Activation('relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Dropout(0.25))\n",
        "\n",
        "model.add(Flatten())\n",
        "model.add(Dense(128))\n",
        "model.add(Activation('relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(10))\n",
        "model.add(Activation('softmax'))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "6fcc147f-94a3-1012-49a1-7966ea4e553e"
      },
      "source": [
        "This Keras function simplifies augmentation, i.e. it randomly modifies the input for training to prevent overfitting. You can also normalize input with this function, just remember to adjust the validation and test sets accordingly in that case."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "17cdb0ec-ba92-6663-b329-0612f6c1c580"
      },
      "outputs": [],
      "source": [
        "datagen = ImageDataGenerator(zoom_range = 0.1,\n",
        "                            height_shift_range = 0.1,\n",
        "                            width_shift_range = 0.1,\n",
        "                            rotation_range = 20)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "fa0d39af-187d-c31d-8c5d-e580f3b34e40"
      },
      "source": [
        "The parameters here were chosen for the Kaggle kernel. For better performance, try reducing the learning rate and increase the number of epochs. I was able to reach 99.5% which put me at place 75 on the leaderboard. With a GPU, it still takes less than an hour to train."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "6e24e63b-a181-2be2-692a-038c3b4ac648"
      },
      "outputs": [],
      "source": [
        "model.compile(loss='categorical_crossentropy', optimizer = Adam(lr=1e-3), metrics = [\"accuracy\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "c4f43457-54c4-daf5-bd3b-4c5456ae7827"
      },
      "source": [
        "EarlyStopping is only important if you plan to run many epochs. ModelCheckpoint is more useful here, as you can resume training after a kernel restart: just run\n",
        "> model.load_weights(''mnist.h5'')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "00497dd9-889c-5de6-030b-a4ce90a1e7c5"
      },
      "outputs": [],
      "source": [
        "callbacks = [EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=2, mode='auto'),\n",
        "            ModelCheckpoint('mnist.h5', monitor='val_loss', save_best_only=True, verbose=0)]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "8fae6b17-d0db-d3b8-24a2-f1fec76508bb"
      },
      "source": [
        "Model training. Note that the training loss is initially much larger than the validation loss, because of Dropout."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c34d0682-2777-382d-ab1c-fa0186fc0ae7"
      },
      "outputs": [],
      "source": [
        "hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size = 64),\n",
        "                           steps_per_epoch = n_train/100, #Take away 100 when not on Kaggle kernel\n",
        "                           epochs = 5, #Increase this when not on Kaggle kernel\n",
        "                           verbose = 2,  #verbose=1 outputs ETA, but doesn't work well in the cloud\n",
        "                           validation_data = (x_val, y_val),\n",
        "                           callbacks = callbacks)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "002ce4ad-9a8f-8a1e-c658-40f9c371f91d"
      },
      "outputs": [],
      "source": [
        "mnist_testset = np.loadtxt(test_file, skiprows=1, dtype='int', delimiter=',')\n",
        "x_test = mnist_testset.astype(\"float32\")/255.0\n",
        "n_test = x_test.shape[0]\n",
        "x_test = x_test.reshape(n_test, 28, 28, 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7413800d-21c3-654f-8b39-27e3df9c4391"
      },
      "outputs": [],
      "source": [
        "y_test = model.predict(x_test, batch_size=64)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "074aabf5-4a9a-bfdb-cbdd-0a8f06dc5fae"
      },
      "source": [
        "y_test consists of class probabilities, I now select the class with highest probability"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "575d9348-99ec-9b8c-f5a5-ceac44c5308d"
      },
      "outputs": [],
      "source": [
        "y_index = np.argmax(y_test,axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "cbf31eb1-2ffd-efb5-e2b2-b4dda99fa1da"
      },
      "outputs": [],
      "source": [
        "with open(output_file, 'w') as f :\n",
        "    f.write('ImageId,Label\\n')\n",
        "    for i in range(0,n_test) :\n",
        "        f.write(\"\".join([str(i+1),',',str(y_index[i]),'\\n']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "580a57a8-c8b2-cb5b-9962-a743f50eb5a4"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}