{"cells":[{"metadata":{"id":"AFijhuh_F0Xl","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.test.gpu_device_name())\n\nimport os\nfrom tensorflow.keras.optimizers import SGD, Adam\nfrom tensorflow.keras.preprocessing.image import img_to_array, ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler, ModelCheckpoint\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.datasets import mnist\nfrom tensorflow.keras import backend as K\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.metrics import classification_report\nfrom sklearn.feature_extraction.image import extract_patches_2d\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib\nimport matplotlib.pyplot as plt\nmatplotlib.use('Agg')     \n%matplotlib inline\n\nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"id":"DhBsRmmDEZX5","colab_type":"text"},"cell_type":"markdown","source":"# Load the MNIST dataset from Keras"},{"metadata":{"id":"cj8pDsPR-M0e","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"(X_train, y_train), (X_ttest, y_ttest) = mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"id":"z59BX2Xv-UqW","colab_type":"code","outputId":"b7948c61-57c4-4af4-d249-68374a5f222e","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"'''\nLet's log the training set shape.\n\nThe first column indicates that we have 60 000 training samples\nand the two last columns indicate that the dimension of each sample is (h, w) <=> (28, 28).\n'''\n\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"G6wlrE6mOy0y","colab_type":"code","outputId":"c2a9fe20-b9db-4628-814e-a3bda404fb88","colab":{"base_uri":"https://localhost:8080/","height":282},"trusted":true},"cell_type":"code","source":"'''\nAs we can see, we've got 60000 samples in our training set and 10000 samples in our testing set.\nEach image, or sample, or datapoint has a dimension of (28, 28) and belongs\nto the gray-scale color-space, so their depth equals 1.\n'''\n\nplt.imshow(X_train[0])\nplt.title(label='Image at position 0')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"1tWY2Cc0PHuF","colab_type":"code","outputId":"f2cbb03e-6b91-49ff-865b-ab864cdc6cd1","colab":{"base_uri":"https://localhost:8080/","height":580},"trusted":true},"cell_type":"code","source":"'''\nAs you can see, images in the MNIST dataset are heavily pre-processed.\nThat's why this is used as a benchmark dataset and getting very high accuracy is common.\nIn a real-world problem, we'll need to do some image pre-processing to enchance images\nand extract meaningful features.\n'''\n\nfig, ax = plt.subplots(\n    nrows=6,\n    ncols=5,\n    figsize=[6, 8]\n)\n\nfor index, axi in enumerate(ax.flat):\n    axi.imshow(X_train[index])\n    axi.set_title(f'Image #{index}')\n\nplt.tight_layout(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"8auyMxrkFkbi","colab_type":"text"},"cell_type":"markdown","source":"# Normalize pixel intensities to range [0, 1]"},{"metadata":{"id":"hTYhdH_Q-fTz","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nNormalization of pixel intensities is adjusting values measured on different scales to a notionally common scale.\nThat's a best practice you have to follow because weights reach optimum values faster.\nTherefore, the network converges faster.\n\nSo, instead of having pixel intensities in the range [0, 255] in the gray-scale color-space,\nwe're going to scale them into the range [0, 1].\nThere are many normalization techniques and this is one of them.\n'''\n\n(X_train, X_ttest) = (child.astype('float32') / 255.0 for child in [X_train, X_ttest])","execution_count":null,"outputs":[]},{"metadata":{"id":"4K03owgxJdqM","colab_type":"text"},"cell_type":"markdown","source":"# Reshape datapoints"},{"metadata":{"id":"x2WE9X7EDidV","colab_type":"code","outputId":"9b8aa7a0-6835-4251-ee09-602b912342f1","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"'''\nReshape the both training and testing set that way \nthe number of samples is the first entry in the matrix,\nthe single channel as the second entry,\nfollowed by the number of rows and columns.\n\n(num_samples, rows, columns, channel)\n'''\n\nX_train = X_train.reshape(-1, 28, 28, 1)\nX_ttest = X_ttest.reshape(-1, 28, 28, 1)\n\n# Let's log the new shape.\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"id":"fzqpG4dBOnNh","colab_type":"text"},"cell_type":"markdown","source":"# One-Hot Encode labels\n"},{"metadata":{"id":"W7rOX0A1Pf_F","colab_type":"code","outputId":"28f890e8-e609-4e7d-8618-a1d7f322cd74","colab":{"base_uri":"https://localhost:8080/","height":419},"trusted":true},"cell_type":"code","source":"'''\nThere are 60000 integers labels for the training set,\neach one corresponding to one single datapoint.\nThat means, for a given Xi datapoint we got a Yi label in the range [0, 9].\nFor instance, the datapoint at the index 59995 of our training set has the label 8.\n'''\n\npd.DataFrame(y_train)","execution_count":null,"outputs":[]},{"metadata":{"id":"5EkoHCr5-vCJ","colab_type":"code","outputId":"a26bc035-1c45-490a-dac4-783b8bb3084a","colab":{"base_uri":"https://localhost:8080/","height":419},"trusted":true},"cell_type":"code","source":"'''\nNow, our previous integer labels are converted to vector labels.\nThis process is called one-hot encoding and most of the machine learning algorithms\nbenefit from this label representation. 2 = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0],\n8 = [0, 0, 0, 0, 0, 0, 0, 0, 1, 0].\n\nWe could also use the to_categorical() function from Keras which yields the exact same values.\n'''\n\nlb = LabelBinarizer()\n(y_train, y_ttest) = (lb.fit_transform(labels) for labels in [y_train, y_ttest])\npd.DataFrame(y_train)","execution_count":null,"outputs":[]},{"metadata":{"id":"Lu9C-aeXP-7P","colab_type":"text"},"cell_type":"markdown","source":"# Data Augmentation"},{"metadata":{"id":"Twnw5D9k-zlE","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nLet's apply some data augmentation.\n\nData augmentation is a set of techniques used to generate new training samples from the original ones\nby applying jitters and perturbations such that the classes labels are not changed.\nIn the context of computer vision, these random transformations can be translating,\nrotating, scaling, shearing, flipping etc.\n\nData augmentation is a form of regularization because the training algorithm is being\nconstantly presented with new training samples,\nallowing it to learn more robust and discriminative patterns\nand reducing overfitting.\n'''\n\ndaug = ImageDataGenerator(\n    featurewise_center=False,\n    samplewise_center=False,\n    featurewise_std_normalization=False,\n    samplewise_std_normalization=False,\n    zca_whitening=False,\n    rotation_range=10,\n    zoom_range = 0.1, \n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    horizontal_flip=False,\n    vertical_flip=False\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"lt3KsDRHQHjw","colab_type":"text"},"cell_type":"markdown","source":"# Build the model"},{"metadata":{"id":"yyv1La7e-8O_","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nHere, we've imported our CNN.\n\nThat's a small VGG-like net, with two stacks of (Conv => ReLU => BN) * 2 => POOL => DO\nand a Fully-Connected layer at the end.\nPay attention to Batch Normalization and Dropout layers\nwhich help to reduce overfitting.\n'''\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import BatchNormalization, Conv2D, MaxPooling2D, Activation, Flatten, Dropout, Dense\n\nclass CustomNet(object):\n    @staticmethod\n    def build(width, height, num_classes, depth=3):\n        model = Sequential()\n        input_shape = (height, width, depth)\n        chan_dim = -1\n        \n        # (Conv => ReLU => BN) * 3 => POOL\n        model.add(Conv2D(64, (3, 3), padding='same', input_shape=input_shape))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(64, (3, 3), padding='same'))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(64, (3, 3), padding='same'))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        \n         # (Conv => ReLU => BN) * 3 => POOL => DO\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(Conv2D(128, (3, 3), padding='same'))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization(axis=chan_dim))\n        model.add(MaxPooling2D(pool_size=(2, 2)))\n        model.add(Dropout(0.25))\n        \n        # FC => ReLU => BN => DO\n        model.add(Flatten())\n        model.add(Dense(256))\n        model.add(Activation('relu'))\n        model.add(BatchNormalization())\n        model.add(Dropout(0.5))\n        \n        # Softmax\n        model.add(Dense(num_classes))\n        model.add(Activation('softmax'))\n        \n        print(model.summary())\n        \n        return model","execution_count":null,"outputs":[]},{"metadata":{"id":"H1gFq6w1_EPG","colab_type":"code","outputId":"7ca261b3-ea81-4904-f85d-dccc4208d39e","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"net = CustomNet()\n\nmodel = net.build(\n    width=28,\n    height=28,\n    num_classes=10,\n    depth=1)","execution_count":null,"outputs":[]},{"metadata":{"id":"KTpNXCOv_MSN","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nWhen the model has seen all of your training samples, we say that one epoch has passed.\nWe're going to train the model for 50 epochs.\n'''\n\nnum_epochs = 50","execution_count":null,"outputs":[]},{"metadata":{"id":"OMvmtL3nQ-Ci","colab_type":"text"},"cell_type":"markdown","source":"# Set the optimizer and hyperparameters"},{"metadata":{"id":"DBEwOLKt_PFC","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nLet's use an optimization method.\n\nOptimization algorithms are the engines that power neural networks and\nenable them to learn patterns from data by tweaking and seeking for optimal weights values.\nMost common one is the (Stochastic) Gradient Descent, but I'll use Adam here.\n\nAs you can see, the first param is lr, or learning rate.\nThis is one of the most important hyperparameters we have to tune.\nA learning rate is the step your optimization algorithm is going to make toward\nthe direction that leads to a lower loss function (and a higher accuracy).\n\nIf the learning rate is too small, the algorithm is going to make tiny steps slowing down the process.\nBut on the other hand, if the learning rate is too high,\nthe algorithm risks to bounce around the loss landscape and not actually “learn” any patterns from your data.\n'''\n\n# Initial learning rate\ninit_lr = 0.001\n\nadam_opt = Adam(\n    lr=init_lr,\n    beta_1=0.9,\n    beta_2=0.999,\n    epsilon=1e-08,\n    decay=0.0\n)\n\n'''\nLet's now define a learn-rate scheduler.\n\nThe decay is used to slowly reduce the learning rate over time.\nDecaying the learning rate is helpful in reducing overfitting \nand obtaining higher classification accuracy – the smaller the learning rate is, \nthe smaller the weight updates will be. \nWe're going to use a polynomial decay. \nAlthough there are many way to do that.\n'''\n\ndef polynomial_decay(epoch):\n    max_epochs = num_epochs\n    base_lr = init_lr\n    power = 2.0\n    \n    return base_lr * (1 - (epoch / float(max_epochs))) ** power","execution_count":null,"outputs":[]},{"metadata":{"id":"UooJ3CYmQwHy","colab_type":"code","outputId":"23be0956-a445-463e-937e-47d736664ce0","colab":{"base_uri":"https://localhost:8080/","height":282},"trusted":true},"cell_type":"code","source":"# Let's plot it.\n\nx = np.linspace(0, num_epochs)\nfx = [init_lr * (1 - (i / float(num_epochs))) ** 2.0 for i in range(len(x))]\nplt.plot(x, fx)\nplt.title(label='Polynomial decay, power 2')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"B7MF62w5RL3Y","colab_type":"text"},"cell_type":"markdown","source":"# Training time!"},{"metadata":{"id":"WTK8tGFp_asi","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"'''\nHere, we define two callbacks.\n\nCallbacks are functions executed at the end of an epoch.\nThe first one save our model (checkpoint) whenever the loss decreases (therefore our accuracy improves).\nThat way we keep the best model. The last one is our learning rate scheduler using the polynomial decay.\n'''\n\n'''\ncheckpointHandler = ModelCheckpoint(\n    os.path.join(base_dir, 'best_c10_weights.hdf5'),\n    monitor='val_loss',\n    save_best_only=True,\n    verbose=1\n)\n'''\n\ncallbacks = [\n    LearningRateScheduler(polynomial_decay)\n    # checkpointHandler\n]","execution_count":null,"outputs":[]},{"metadata":{"id":"af9qwYmE_dVT","colab_type":"code","outputId":"903d6709-211e-4324-b1ec-2340d77c2f25","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":true},"cell_type":"code","source":"batch_size = 128\n\nprint('# Compiling the model...')\nmodel.compile(\n    loss='categorical_crossentropy',\n    optimizer=adam_opt,\n    metrics=['accuracy']\n)\n\nprint('# Training the network...')\nh = model.fit_generator(\n    daug.flow(X_train, y_train, batch_size=batch_size),\n    validation_data=(X_ttest, y_ttest),\n    epochs=num_epochs,\n    steps_per_epoch=len(X_train) // batch_size,\n    callbacks=callbacks,\n    verbose=1\n)","execution_count":null,"outputs":[]},{"metadata":{"id":"zkTCiSyBRb1n","colab_type":"text"},"cell_type":"markdown","source":"# Evaluate"},{"metadata":{"id":"oXQ4M6Lz_eAb","colab_type":"code","outputId":"0ed3335b-b462-4034-fd09-133202682e36","colab":{"base_uri":"https://localhost:8080/","height":323},"trusted":true},"cell_type":"code","source":"label_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n\nprint('Confusion matrix:')\npreds = model.predict(X_ttest, batch_size=batch_size)\nprint(classification_report(y_ttest.argmax(axis=1),\npreds.argmax(axis=1), target_names=label_names))","execution_count":null,"outputs":[]},{"metadata":{"id":"mAd1OXGtReYN","colab_type":"text"},"cell_type":"markdown","source":"# Visualize curves"},{"metadata":{"id":"xGB-cLnW_toq","colab_type":"code","outputId":"e398aee0-f4dc-41dd-f86e-4aeb0d1a2675","colab":{"base_uri":"https://localhost:8080/","height":626},"trusted":true},"cell_type":"code","source":"# Loss\nplt.figure(figsize=(8, 5))\nplt.plot(np.arange(0, num_epochs), h.history['loss'], label='train_loss')\nplt.plot(np.arange(0, num_epochs), h.history['val_loss'], label='val_loss')\nplt.title('Loss')\nplt.legend()\nplt.show()\n\n# Accuracy\nplt.figure(figsize=(8, 5))\nplt.plot(np.arange(0, num_epochs), h.history['accuracy'], label='train_acc')\nplt.plot(np.arange(0, num_epochs), h.history['val_accuracy'], label='val_acc')\nplt.title('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"id":"NQ5OuM1ONtMV","colab_type":"text"},"cell_type":"markdown","source":"# Kaggle"},{"metadata":{"id":"7MI8GAR1IDXq","colab_type":"code","outputId":"7a013f0e-63ac-4e4b-808d-5ce0ec8373d5","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":true},"cell_type":"code","source":"'''\nPushing a submission to Kaggle.\n\nFirst of, load the test set, normalize it and reshape it.\nThen, make predictions via the trained model and build the\nsubmission csv file.\n'''\n\nsub_X_test = pd.read_csv('../input/digit-recognizer/test.csv')   # Load CSV\nsub_X_test = sub_X_test.iloc[:,:].values                         # Get raw pixel intensities\nsub_X_test = sub_X_test.reshape(sub_X_test.shape[0], 28, 28, 1)  # Reshape to meet Keras requirements\nsub_X_test = sub_X_test / 255.0                                  # Normalize to range [0, 1]\n# Get predictions\npreds = model.predict_classes(sub_X_test, batch_size=batch_size)\n# Generate a submission file\nid_col = np.arange(1, preds.shape[0] + 1)\nsubmission = pd.DataFrame({'ImageId': id_col, 'Label': preds})                                         # Shift index\nprint(pd.DataFrame(submission))\nsubmission.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"mnist_colab.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}