{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "fc951ae3-f83f-bfa7-1249-939cde8f2dbd"
      },
      "source": [
        "Copied verbatim from Eike Dehling.  I will experiment with it later."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ab283bde-3299-d479-cf7f-331772eb448c"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "47493020-a926-fde5-3475-2b6c1b3b7ce9"
      },
      "outputs": [],
      "source": [
        "from keras.utils.np_utils import to_categorical\n",
        "from keras import backend as K"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "feca8e71-39a7-a662-f3c7-9ae14e747b2b"
      },
      "source": [
        "Now we load & preprocess the training data. I borrowed this from other notebooks.."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "4342dbd0-5fb6-8ce0-eb30-12fe658386aa"
      },
      "outputs": [],
      "source": [
        "K.set_image_dim_ordering('th') #input shape: (channels, height, width)\n",
        "\n",
        "train_df = pd.read_csv(\"../input/train.csv\")\n",
        "valid_df = pd.read_csv(\"../input/test.csv\")\n",
        "\n",
        "x_train = train_df.drop(['label'], axis=1).values.astype('float32')\n",
        "Y_train = train_df['label'].values\n",
        "x_valid = valid_df.values.astype('float32')\n",
        "\n",
        "img_width, img_height = 28, 28\n",
        "\n",
        "n_train = x_train.shape[0]\n",
        "n_valid = x_valid.shape[0]\n",
        "\n",
        "x_train = x_train.reshape(n_train,1,img_width,img_height)\n",
        "x_valid = x_valid.reshape(n_valid,1,img_width,img_height)\n",
        "\n",
        "x_train = x_train/255 #normalize from [0,255] to [0,1]\n",
        "x_valid = x_valid/255\n",
        "\n",
        "y_train = to_categorical(Y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "21687abb-6b29-d108-cf09-cabdd4c86f93"
      },
      "outputs": [],
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers.convolutional import *\n",
        "from keras.layers.core import Dropout, Dense, Flatten"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "617f72c7-959c-c4dd-5a33-35d35494f9f6"
      },
      "source": [
        "Here we define the models layers ; for me two convolutional layers, two dense layers and dropout/regularizing at the end produced the best results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d88ac290-7719-38c7-77f5-4feaa9520409"
      },
      "outputs": [],
      "source": [
        "model = Sequential()\n",
        "model.add(Conv2D(filters=40, kernel_size=(5,5), activation='relu', batch_input_shape=(None, 1, img_width, img_height)))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Conv2D(filters=40, kernel_size=(5,5), activation='relu'))\n",
        "model.add(MaxPooling2D(pool_size=(2,2)))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(1000, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(1000, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(10, activation='softmax', activity_regularizer='l1_l2'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "74217365-ed8a-c7a6-eaa3-d76a17b82e54"
      },
      "outputs": [],
      "source": [
        "model.compile(optimizer='adam',\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['accuracy'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bf662e3e-da7e-31c3-6e3f-a1903f4adefa"
      },
      "outputs": [],
      "source": [
        "model.fit(x_train,\n",
        "          y_train,\n",
        "          batch_size=128,\n",
        "          epochs=1,  # For running on your own laptop, ~ 15 epochs produces much better results but takes longer\n",
        "          verbose=2,\n",
        "          validation_split=.2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "74f95c7f-db84-8669-3420-14665c923ad1"
      },
      "source": [
        "To generate and output the predictions, run the code below!"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "2d31f8ae-576b-8211-49ac-d61b340564a8"
      },
      "outputs": [],
      "source": [
        "yPred = model.predict_classes(x_valid,batch_size=32,verbose=1)\n",
        "np.savetxt('predictions.csv', np.c_[range(1,len(yPred)+1),yPred], delimiter=',', header = 'ImageId,Label', comments = '', fmt='%d')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "c2cf6332-0094-55c9-9d27-104b4091c0e5"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}