{"cells":[{"metadata":{},"cell_type":"markdown","source":"Based on 'Keras CNN with over 99% accuracy' kernel"},{"metadata":{"trusted":false},"cell_type":"code","source":"import numpy as np\n\nfrom keras.utils import np_utils\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Activation, Conv2D, MaxPooling2D\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.layers.normalization import BatchNormalization","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"train_file = \"input/train.csv\"\ntest_file = \"input/test.csv\"\noutput_file = \"submission.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"mnist_dataset = np.loadtxt(train_file, skiprows=1, dtype='int', delimiter=',')","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"val_split = 0.125\nn_raw = mnist_dataset.shape[0]\nn_val = int(n_raw * val_split + 0.5)\nn_train = n_raw - n_val\n\nnp.random.shuffle(mnist_dataset)\nx_val, x_train = mnist_dataset[:n_val,1:], mnist_dataset[n_val:,1:]\ny_val, y_train = mnist_dataset[:n_val,0], mnist_dataset[n_val:,0]\n\nx_train = x_train.astype(\"float32\")/255.0\nx_val = x_val.astype(\"float32\")/255.0\ny_train = np_utils.to_categorical(y_train)\ny_val = np_utils.to_categorical(y_val)\n\nn_classes = y_train.shape[1]\nx_train = x_train.reshape(n_train, 28, 28, 1)\nx_val = x_val.reshape(n_val, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Feel free to modify the layers."},{"metadata":{"trusted":false},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (3, 3), input_shape = (28, 28, 1), \n                 padding='same', kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3, 3), \n                 padding='same',kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(filters = 128, kernel_size = (3, 3),\n                 padding='same',kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\n\nmodel.add(Conv2D(filters = 128, kernel_size = (3, 3),\n                 padding='same',kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(1024, kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(256, kernel_initializer='he_normal'))\nmodel.add(Activation('elu'))\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(10, kernel_initializer='he_normal'))\nmodel.add(Activation('softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This Keras function simplifies augmentation, i.e. it randomly modifies the input for training to prevent overfitting. You can also normalize input with this function, just remember to adjust the validation and test sets accordingly in that case."},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"datagen = ImageDataGenerator(zoom_range = 0.1,\n                            height_shift_range = 0.1,\n                            width_shift_range = 0.1,\n                            rotation_range = 20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The parameters here were chosen for the Kaggle kernel. For better performance, try reducing the learning rate and increase the number of epochs. I was able to reach 99.5% which put me at place 75 on the leaderboard. With a GPU, it still takes less than an hour to train."},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"model.compile(loss='categorical_crossentropy', optimizer = Adam(lr=5e-4), metrics = [\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"EarlyStopping is only important if you plan to run many epochs. ModelCheckpoint is more useful here, as you can resume training after a kernel restart: just run\n> model.load_weights(''mnist.h5'')"},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"callbacks = [EarlyStopping(monitor='val_loss', min_delta=0, patience=2, verbose=2, mode='auto'),\n            ModelCheckpoint('mnist.h5', monitor='val_loss', save_best_only=True, verbose=0)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Model training. Note that the training loss is initially much larger than the validation loss, because of Dropout."},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"read_model = False\nif read_model:\n    if os.path.isfile(\"output/model.h5\"):\n        model = load_model(\"output/model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size = 2048),\n                           steps_per_epoch = n_train/2048, \n                           epochs = 10, \n                           verbose = 1,  \n                           validation_data = (x_val, y_val),\n                           callbacks = callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.models import load_model\n\n# save json\njson_model = model.to_json()\nwith open(\"output/model.json\", \"w\") as json_file:\n    json_file.write(json_model)\n# save model\nmodel.save(\"output/model.h5\")","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"mnist_testset = np.loadtxt(test_file, skiprows=1, dtype='int', delimiter=',')\nx_test = mnist_testset.astype(\"float32\")/255.0\nn_test = x_test.shape[0]\nx_test = x_test.reshape(n_test, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"y_test = model.predict(x_test, batch_size=128)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"y_test consists of class probabilities, I now select the class with highest probability"},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"y_index = np.argmax(y_test,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"with open(output_file, 'w') as f :\n    f.write('ImageId,Label\\n')\n    for i in range(0,n_test) :\n        f.write(\"\".join([str(i+1),',',str(y_index[i]),'\\n']))","execution_count":null,"outputs":[]},{"metadata":{"collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}