{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Data Loading"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\n\n# Separate training X's and Y's\nX_train = train.drop(labels=[\"label\"], axis=1)\nY_train = train['label']\ndel train\n\n# Normalize pixel values\nX_train = X_train / 255.\ntest = test / 255.\n\n# Reshape X's into 28x28 images\nX_train = X_train.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1, 28, 28, 1)\n\n# One-hot encode labels\n#Y_train = to_categorical(Y_train, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, Y_train, Y_val = train_test_split(\n    X_train,\n    Y_train,\n    test_size=0.1,\n    random_state=42\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n    rotation_range=10,\n    zoom_range=(1.15, 0.95),\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=False,\n    vertical_flip=False,\n    #brightness_range=[0.7,1.1],\n    shear_range=5\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Note!**\nThe objective of data augmentation is to be able to generate more data then what we have, but that data has to effectively be indistinguishable from the original training data and fall within the same distribution, otherwhise the model could learn to fit the augmented (training) data (**overfitting**) and fail to predict correctly on the original data itself."},{"metadata":{},"cell_type":"markdown","source":"## Inspect Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig, axs = plt.subplots(5,10)\n\nimgs = X_train[:25]\nimgs_aug = datagen.flow(imgs, batch_size=25, shuffle=False).next()\n\ni = 0\n\nfor x in range(5):\n    for y in range(5):\n        axs[y,x*2].imshow(1-imgs[i][:,:,0], cmap='gray')\n        axs[y,x*2].axis('off')\n        axs[y,x*2+1].imshow(1-imgs_aug[i][:,:,0], cmap='gray')\n        axs[y,x*2+1].axis('off')\n        i += 1\n        \nplt.subplots_adjust(wspace=0, hspace=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import Sequential\nfrom keras.layers import Conv2D, Dense, Dropout, Flatten, BatchNormalization, LeakyReLU\nfrom keras.initializers import RandomNormal\n\ninit = RandomNormal(stddev=0.02)\n\nmodel = Sequential([\n    Conv2D(32, 3, input_shape=(28, 28, 1), activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    \n    Conv2D(32, 3, activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    \n    Conv2D(32, 5, strides=2, padding='same', activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    Dropout(0.4),\n    \n    Conv2D(64, 3, activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    \n    Conv2D(64, 3, activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    \n    Conv2D(64, 5, strides=2, padding='same', activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    Dropout(0.4),\n    \n    Conv2D(128, 4, activation='relu', kernel_initializer=init),\n    BatchNormalization(),\n    \n    Flatten(),\n    Dropout(0.4),\n    Dense(10, activation='softmax')\n])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.optimizers import Adam\n\nmodel.compile(\n    optimizer=Adam(lr=1e-3),\n    loss='sparse_categorical_crossentropy',\n    metrics=['sparse_categorical_accuracy']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\n\nlearning_rate_reduction = ReduceLROnPlateau(\n    monitor='val_sparse_categorical_accuracy', \n    patience=3, \n    verbose=1, \n    factor=0.5, \n    min_lr=0.00001\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(\n    datagen.flow(X_train, Y_train, batch_size=64),\n    epochs=45,\n    validation_data=(X_val, Y_val),\n    callbacks=[learning_rate_reduction],\n    use_multiprocessing=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\")\nlegend = ax[0].legend(loc='best')\n\nax[1].plot(history.history['sparse_categorical_accuracy'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_sparse_categorical_accuracy'], color='r',label=\"Validation accuracy\")\nlegend = ax[1].legend(loc='best')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\nresults = model.predict(test)\nresults = np.argmax(results, axis=1)\n\nsubmission = pd.concat([\n    pd.Series(range(1,28001), name=\"ImageId\"),\n    pd.Series(results, name=\"Label\")\n], axis=1)\n\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}