{"metadata": {"language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "mimetype": "text/x-python", "pygments_lexer": "ipython3", "name": "python", "file_extension": ".py", "version": "3.6.1"}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "nbformat": 4, "nbformat_minor": 0, "cells": [{"metadata": {"_uuid": "839bb24a36d7bac088dc05fd1d0d5ece51e6bdb9", "_cell_guid": "4f2fc7f7-77d8-4290-ae70-c7d78dbe7f76", "collapsed": false, "_execution_state": "idle"}, "source": "Hey! That's my hello world DenseNet-ish MNIST kernel. So far, I've achieved 99,429% on test.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "1bfb47a6a27ab74b43aaf0eb9c67e429d026087f", "_cell_guid": "99ae7116-27c0-4428-8c41-75296948c0d4", "collapsed": false, "_execution_state": "idle"}, "source": "First, lets import all the necessary modules.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "bcfbaf8df4f2a1044cf753fe0aaf9a7a3f51b97b", "trusted": false, "_cell_guid": "0b9d7be2-8637-4dcb-99b6-84ceeefbf692", "_execution_state": "idle"}, "source": "import numpy as np\nimport pandas as pd\nimport os, shutil\nfrom keras.models import Model, load_model\nfrom keras.layers import (Input, Concatenate, Dense, \n                          Flatten, Dropout, Conv2D, \n                          AveragePooling2D, GaussianNoise,\n                          BatchNormalization)\nfrom keras.initializers import RandomNormal\nfrom keras.optimizers import SGD\nfrom keras.datasets import mnist\nfrom keras.utils import to_categorical\nfrom keras.regularizers import l2\nfrom keras.callbacks import ModelCheckpoint\nimport keras.backend as K", "outputs": [], "cell_type": "code", "execution_count": 1}, {"metadata": {"_uuid": "2a70c20fee964bc58e3fd71a2e08cf150c48232c", "_cell_guid": "c992d086-33a1-4873-ad59-4552726e244d", "trusted": false, "collapsed": false, "_execution_state": "idle"}, "source": "Next, define our DenseNet.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "8df27fc6493e4775b44c40ee38743e333b03fd09", "_cell_guid": "cfba2b2f-9986-4940-902e-61af33de1641", "trusted": false, "collapsed": false, "_execution_state": "idle"}, "source": "'''\n    Constucts single DenseNet block\n    \n    @param inputs - input 4D tensor\n    @param n - number of dense units in block\n    @param k - number of dense unit filters\n    @param b - bottleneck factor of dense unit\n'''\ndef DenseBlock(inputs, n = 4, k = 12, b = 2):\n    for i in range(n):\n        cur_layer = Conv2D(\n            filters = b * k,\n            kernel_size = (1, 1),\n            activation = 'relu'\n        )(inputs)\n        cur_layer = Conv2D(\n            filters = k,\n            padding = 'same',\n            kernel_size = (3, 3),\n            activation = 'relu'\n        )(cur_layer)\n        cur_layer = Dropout(0.2)(cur_layer)\n        inputs = Concatenate(\n            axis = 3\n        )([inputs, cur_layer])\n    return inputs\n\n'''\n    Constucts DenseNet\n'''\ndef DenseNet(k = 12):\n    inputs = Input(shape = [28, 28, 1])\n    cur_layer = Conv2D(\n        filters = 2 * k,\n        kernel_size = (5, 5),\n        activation = 'relu'\n        #kernel_regularizer = l2(0.05)\n    )(inputs)\n    cur_layer = AveragePooling2D(\n        pool_size = (2, 2)\n    )(cur_layer)\n    # block 1\n    cur_layer = DenseBlock(cur_layer, 4, k, 4)\n    cur_layer = BatchNormalization()(cur_layer)\n    cur_layer = Conv2D(\n        filters = int(cur_layer.get_shape().as_list()[-1] / 2),\n        kernel_size = (1, 1),\n        activation = 'relu'\n    )(cur_layer)\n    cur_layer = AveragePooling2D(\n        pool_size = (2, 2)\n    )(cur_layer)\n    # block 2\n    cur_layer = DenseBlock(cur_layer, 8, k, 4)\n    cur_layer = BatchNormalization()(cur_layer)\n    cur_layer = Conv2D(\n        filters = int(cur_layer.get_shape().as_list()[-1] / 2),\n        kernel_size = (1, 1),\n        activation = 'relu'\n    )(cur_layer)\n    cur_layer = AveragePooling2D(\n        pool_size = (2, 2)\n    )(cur_layer)\n    # block 3\n    cur_layer = DenseBlock(cur_layer, 8, k, 4)\n    cur_layer = BatchNormalization()(cur_layer)\n    cur_layer = Conv2D(\n        filters = int(cur_layer.get_shape().as_list()[-1] / 2),\n        kernel_size = (1, 1),\n        activation = 'relu'\n    )(cur_layer)\n    cur_layer = AveragePooling2D(\n        pool_size = (3, 3)\n    )(cur_layer)\n    # final\n    cur_layer = Flatten()(cur_layer)\n    cur_layer = Dense(\n        units = 10,\n        activation = 'softmax'\n    )(cur_layer)\n    model = Model(\n        inputs = inputs,\n        outputs = cur_layer\n    )\n    return model", "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "8c78b441cbe702a7cf07bf1dd4af38774fc02844", "_cell_guid": "4dc560bf-b2ca-463c-ac4f-b608f3cea66e", "collapsed": false, "_execution_state": "idle"}, "source": "Read data and create model.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "7d288a0c0333b1eabd045e84230fd00619b9351f", "_cell_guid": "c32d9cec-c9ee-40ca-981e-8bcdf9d5552e", "trusted": false, "collapsed": false, "_execution_state": "idle"}, "source": "train_data = pd.read_csv('../input/train.csv', delimiter=',')\nX_train = np.reshape(train_data.loc[:, 'pixel0':].as_matrix(), [-1, 28, 28, 1]) / 255.0\ny_train = train_data.loc[:, 'label']\ny_train = to_categorical(y_train.as_matrix())\ntest_data = pd.read_csv('../input/test.csv', delimiter=',')\nX_test = np.reshape(test_data.loc[:, 'pixel0':].as_matrix(), [-1, 28, 28, 1]) / 255.0\n\nmodel = DenseNet(k = 24)", "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "e185c39d8b98a3aa3db1dac1128d2642dd0d70e6", "_cell_guid": "73c76bee-ddd3-4c41-9bbf-34bcca955345", "collapsed": false, "_execution_state": "idle"}, "source": "First, train model for 5 epochs with 0.1 learning rate, then 10 more with 0.01.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "ee565097aa79c954ff9d9de3155b63b886031dd7", "_cell_guid": "0f492d42-5bb8-43b3-8b00-859b3c263c23", "trusted": false, "collapsed": false, "_execution_state": "idle"}, "source": "opt = SGD(\n    lr = 0.1,\n    decay = 1e-04,\n    momentum = 0.9,\n    nesterov = True\n)\nmodel.compile(\n    loss = 'categorical_crossentropy', \n    optimizer = opt, \n    metrics = ['categorical_accuracy']\n)\nmodel.fit(\n    x = X_train, \n    y = y_train, \n    epochs = 5, \n    batch_size = 128, \n    verbose = 1\n)", "outputs": [], "execution_count": null, "cell_type": "code"}, {"metadata": {"_uuid": "e4e2185ffe020e5fcb65672e862635932fb09f26", "_cell_guid": "6886fa69-f9f1-46aa-9472-b6e4e3b0d3cb", "collapsed": false, "_execution_state": "idle"}, "source": "That's it! Time to gather predictions.", "outputs": [], "execution_count": null, "cell_type": "markdown"}, {"metadata": {"_uuid": "d4544058e58d3f8d129494643ad274690e59aebe", "_cell_guid": "e7e2148d-a306-4a66-ba5c-0a66ea4216d4", "trusted": false, "collapsed": false, "_execution_state": "idle"}, "source": "preds = model.predict(X_test)", "outputs": [], "execution_count": null, "cell_type": "code"}]}