{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom functools import partial\n\nfrom keras import Model, backend as K\nfrom keras.layers import *\nfrom keras.optimizers import SGD\nfrom keras.callbacks import ModelCheckpoint, LambdaCallback\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import to_categorical\n\nfrom sklearn.model_selection import train_test_split","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"seed = 69\nnp.random.seed(seed)\n\nepochs = 30\nbatch_size = 256\ndropout = .5\nnoise = .3\nalpha = 1 # Variable used to decay dropout and data augmentation during training.\nminalpha = .2\n\ndf_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\ndf_test = pd.read_csv('../input/Kannada-MNIST/Dig-MNIST.csv')\n\nx_train, y_train = np.array(df_train.iloc[:, 1:]), np.array([i for i in df_train.iloc[:, 0].apply(to_categorical, args=(10,))])\n# x_test, y_test = np.array(df_test.iloc[:, 1:]), np.array([i for i in df_test.iloc[:, 0].apply(to_categorical, args=(10,))])\nx_train, x_test, y_train, y_test = train_test_split(x_train, y_train, test_size=.4, random_state=seed)\n\n# Reshape images and map values between -1 and 1.\nx_train = x_train.reshape((x_train.shape[0], 28, 28, 1))\nx_test = x_test.reshape((x_test.shape[0], 28, 28, 1))\n\ndel df_train, df_test","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See https://github.com/keras-team/keras/issues/8826 for why this is necessary.\nclass VariableDropout(Layer):\n    def __init__(self, rate, noise_shape=None, seed=None, **kwargs):\n        self.rate = K.variable(rate)\n        self.noise_shape = noise_shape\n        self.seed = seed\n        super(VariableDropout, self).__init__(**kwargs)\n\n    def call(self, x, training=None):\n        def dropout():\n            return K.dropout(x,\n                             self.rate,\n                             self._get_noise_shape(x),\n                             seed=self.seed)\n            \n        return K.in_train_phase(dropout, x,\n                                training=training)\n    \n    def compute_output_shape(self, input_shape):\n        return input_shape\n        \n    def set_rate(self, rate):\n        K.set_value(self.rate, max(0., rate))\n        \n    def _get_noise_shape(self, inputs):\n        if self.noise_shape is None:\n            return self.noise_shape\n\n        symbolic_shape = K.shape(inputs)\n        noise_shape = [symbolic_shape[axis] if shape is None else shape\n                       for axis, shape in enumerate(self.noise_shape)]\n        \n        return tuple(noise_shape)\n\nclass VariableGaussianNoise(Layer):\n    def __init__(self, stddev, seed=None, **kwargs):\n        self.stddev = K.variable(max(0., stddev))\n        self.seed = seed\n        super(VariableGaussianNoise, self).__init__(**kwargs)\n        \n    def call(self, x, training=None):\n        def noise():\n            return x+K.random_normal(K.shape(x), stddev=self.stddev, seed=self.seed)\n        return K.in_train_phase(noise, x, training=training)\n  \n    def compute_output_shape(self, input_shape):\n        return input_shape\n      \n    def set_stddev(self, stddev):\n        K.set_value(self.stddev, max(0., stddev))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ConvLayer = partial(Conv2D, kernel_size=3, activation='relu', kernel_initializer='orthogonal')\n\ndef model():\n    x = Input((28, 28, 1))\n    y = VariableGaussianNoise(noise, seed=seed, name='noise')(x)\n    y = ConvLayer(32)(y)\n    y = MaxPool2D()(y)\n    y = BatchNormalization()(y)\n    y = ConvLayer(64)(y)\n    y = MaxPool2D()(y)\n    y = BatchNormalization()(y)\n    y = ConvLayer(128)(y)\n    y = GlobalAvgPool2D()(y)\n    y = VariableDropout(dropout, seed=seed, name='dropout')(y)\n    y = Dense(512, activation='relu')(y)\n    y = Dense(10, activation='softmax')(y)\n    \n    return Model(x, y)\n\nmodel = model()\nmodel.compile(SGD(.005, momentum=.9, nesterov=True), 'categorical_crossentropy', metrics=['acc'])\nmodel.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_idg = ImageDataGenerator(rotation_range=.25*alpha,\n                               width_shift_range=.2*alpha,\n                               height_shift_range=.2*alpha,\n                               zoom_range=.2*alpha,\n                               brightness_range=(1-.3*alpha, 1+.3*alpha),\n                               preprocessing_function=lambda x: x/127.5-1)\ntrain_idg = train_idg.flow(x_train,\n                           y_train,\n                           batch_size,\n                           shuffle=False,\n                           seed=seed)\n    \ntest_idg = ImageDataGenerator(preprocessing_function=lambda x: x/127.5-1)\ntest_idg = test_idg.flow(x_test,\n                         y_test,\n                         batch_size=batch_size,\n                         shuffle=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjustAlpha(epoch, logs):\n    global model, alpha, minalpha, dropout, noise\n    alpha = max(minalpha, alpha*.9)\n    model.get_layer('dropout').set_rate(dropout*alpha)\n    model.get_layer('noise').set_stddev(noise*alpha)\n    \nadjust_alpha = LambdaCallback(on_epoch_end=adjustAlpha)\nmodel_checkpoint = ModelCheckpoint('weights.h5', 'val_acc', save_best_only=True, save_weights_only=True, mode='max')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(train_idg,\n                              len(train_idg),\n                              epochs,\n                              callbacks=[adjust_alpha, model_checkpoint],\n                              validation_data=test_idg,\n                              validation_steps=len(test_idg),\n                              shuffle=False).history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Best accuracy: %.2f' % (max(history['val_acc'])*100), '%', sep='')\n\nplt.figure(figsize=(8, 6))\nplt.plot(history['loss'], 'b-', label='loss')\nplt.plot(history['val_loss'], 'r-', label='val_loss')\nplt.title('Losses')\nplt.legend()\nplt.show()\n\nplt.figure(figsize=(8, 6))\nplt.plot(history['acc'], 'b-', label='acc')\nplt.plot(history['val_acc'], 'r-', label='val_acc')\nplt.title('Accuracies')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_weights('weights.h5')\nsample_submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\ndf_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\ndf_test = np.array(df_test.iloc[:, 1:]).reshape((df_test.shape[0], 28, 28, 1))/127.5-1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(df_test.shape[0]):\n    sample_submission['label'][i] = np.argmax(model.predict(df_test[i:i+1]))\nsample_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}