{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\n**My notebook for recognising handwritten digits from the MNIST dataset using a CNN with adaptive learning and callback function to maximise learning.** \n\n**Achieves a score 0.995 with 30 epochs - approx 5 min training time with GPU**\n\n**1. Data pre-processing:**\n* Loading data files\n* Splitting data into train, validation, and test sets\n* Reshaping dimensions of raw data\n\n**2. Digit visualisation:**\n* Function for plotting training data subfigures with true label\n\n**3. Data Processing with Generators:**\n* Train data generator: Augmentations and normalisation\n* Validation data generator: Normalisation\n\n**4. Defining the model:**\n* Callback function\n* Adaptive learning rate\n* Model architecture\n* Model compiling\n\n**5. Training**\n* Model Training and validation\n* Plotting metrics change during training\n\n**6. Testing**\n* Using trained model to make predictions on the test set\n* Exporting predictions as submission csv file\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom keras_preprocessing.image import ImageDataGenerator\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.optimizers import RMSprop","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Data pre-processing:","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Loading data\nroot = \"../input/digit-recognizer/\"\ndata = pd.read_csv(root + \"train.csv\")\ntest = pd.read_csv(root + \"test.csv\")\nsample_submission = pd.read_csv(root + \"sample_submission.csv\")\n\n# Partition the training and validation sets \ntrain_split = int(len(data)*0.9)\ntrain = data[:train_split]\ntrain_labels = train['label'].values.astype('float32')\ntrain = train.drop('label', axis=1)\nvalid = data[train_split:]\nvalid_labels = valid['label'].values.astype('float32')\nvalid = valid.drop('label', axis=1)\n\n# Reshaping the data to 28x28 images\nlen_train, len_valid, len_test = len(train), len(valid), len(test)\nshape_train, shape_valid, shape_test = train.shape, valid.shape, test.shape\nprint(\"training: {}, validation: {}, testing: {}\".format(len_train, len_valid, len_test))\nprint(\"dataset shapes; training: {}, validation: {}, testing: {}\".format(shape_train, shape_valid, shape_test))\n\ntrain_data = train.values.reshape(len_train, 28, 28, 1)\nvalid_data = valid.values.reshape(len_valid, 28, 28, 1)\ntest_data = test.values.reshape(len_test, 28, 28, 1)\n\nreshape_train, reshape_valid, reshape_test = train_data.shape, valid_data.shape, test_data.shape\nprint(\"dataset dimensions after reshaping; training: {}, validation: {}, testing: {}\".format(reshape_train, reshape_valid, reshape_test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **2. Digit visualisation:**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plots digits\ndef plot_digits(data, labels, num_images, columns=3):\n    \"\"\" Plots MNIST digits in a subplot with the label of each being its title.\n\n    Args:\n        data: Matrix of dimensions [number of images, height, width, channels]\n        labels: List of class labels.\n        num_images: Number of images to be plotted\n        columns: Number of columns to display in the figure.\n\n    \"\"\"\n    images = data[:num_images]\n    labels = labels[:num_images]\n    rows = num_images // columns\n    rows += num_images % columns\n    position = range(1, num_images + 1)\n    \n    fig = plt.figure(figsize=(8,8))\n    \n    for n in range(num_images):\n        ax = fig.add_subplot(rows, columns, position[n])\n        ax.imshow(images[n][:, :, 0], cmap = 'Greys')\n        ax.title.set_text(str(labels[n]))\n        ax.axes.xaxis.set_visible(False)\n        ax.axes.yaxis.set_visible(False)\n        plt.tight_layout()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_digits(train_data, train_labels, 15, columns=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Data Processing with Generators:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Passes train dataset through ImageDataGenerator with normalisation and augmentations\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=15,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    shear_range=0.1,\n    zoom_range=0.1,\n    fill_mode='nearest')\n\ntrain_generator = train_datagen.flow(\n    train_data,\n    train_labels,\n    batch_size=512)\n\n# Passes valid dataset through ImageDataGenerator with normalisation \nvalid_datagen = ImageDataGenerator(\n    rescale=1./255)\n\nvalid_generator = valid_datagen.flow(\n    valid_data,\n    valid_labels,\n    batch_size=512)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Defining the model:","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model\n# Creating a callback to end training before overfitting\nclass MyCallBack(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epochs, logs={}):\n        if logs.get('accuracy') > 0.998:\n            print(\"Reached accuracy target, training stopped\")\n            self.model.stop_training = True\n            \n# Adaptive learning \nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_accuracy', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\n# Defining the model\ncall_back = MyCallBack()\nmodel = tf.keras.models.Sequential([\n    # Fist Convolutional Layer\n    tf.keras.layers.Conv2D(64, (5,5), activation='relu', input_shape=(28, 28, 1)),\n    tf.keras.layers.MaxPool2D(2,2),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dropout(0.2),\n    # Second Convolutional Layer\n    tf.keras.layers.Conv2D(128, (3,3), activation='relu'),\n    tf.keras.layers.MaxPool2D(2,2),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dropout(0.2),\n    # Densely Connected Layers\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(10, activation='softmax')\n])\n\n# Compiling the model and displaying architecture summary\nmodel.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 5. Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training\nhistory = model.fit(train_generator, steps_per_epoch=len(train_data)/512, epochs=30, validation_data=(valid_generator), validation_steps=len(valid_data)/512, callbacks=[learning_rate_reduction, call_back])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Extract metrics\nacc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\n\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nplt.figure(figsize=(16, 8))\n\n# Plot accuracy\nplt.subplot(1, 2, 1)\nplt.plot(acc, label='Training Accuracy')\nplt.plot(val_acc, label='Validation Accuracy')\nplt.legend(loc='lower right', frameon=False, fontsize=12)\nplt.ylabel('Accuracy', fontsize= 15)\nplt.ylim([min(plt.ylim()),1])\nplt.title('Training and Validation Accuracy', fontsize= 20)\nplt.xlabel('epoch', fontsize= 15)\nplt.xticks(fontsize=14)\nplt.yticks(fontsize=14)\n\n# Plot loss\nplt.subplot(1, 2, 2)\nplt.plot(loss, label='Training Loss')\nplt.plot(val_loss, label='Validation Loss')\nplt.legend(loc='upper right', frameon=False, fontsize=12)\nplt.ylabel('Binary Cross Entropy', fontsize= 15)\nplt.ylim([0,1.0])\nplt.title('Training and Validation Loss', fontsize= 20)\nplt.xlabel('epoch', fontsize= 15)\nplt.xticks(fontsize=14)\nplt.yticks(fontsize=14)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 6. Testing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Making predictions on the test set\ntest_data = test_data/255\npredictions = model.predict(test_data)\npredictions = np.argmax(predictions, axis=1)\nsample_submission['Label'] = predictions\n\n#Output predictions as csv for competition submission\nsample_submission.to_csv(\"output.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Achieves a score 0.995 with 30 epochs - approx 5 min training time with GPU**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}