{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Using a committee of 15 Convolutional Neural Networks to classify handwritten digits\n","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport matplotlib.pyplot as plt\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.utils.np_utils import to_categorical\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Read the data\ntrain = pd.read_csv('../input/digit-recognizer/train.csv').to_numpy()\ntest = pd.read_csv('../input/digit-recognizer/test.csv').to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separate the images from their labels\nX = train[:, 1:]\ny = train[:, 0]\n\n# Normalize the image data\nX = X / 255\ntest = test / 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X.reshape(42000, 28, 28, 1) # Set the data to be the right shape for the NN\nyh = to_categorical(y, 10) # One-hot encode the labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add data augmentation\nidg = ImageDataGenerator(rotation_range=15, width_shift_range=4, height_shift_range=4)\nidg.fit(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ensemble = []\n# Make 15 CNNs.\n# The CNNs consist of 6 convolutional layers, 2 of which (with strides=2) perform downsampling. These are followed by a fully-connected layer and an output layer.\n# They use ReLU as the activation function (except for the output layer). The CNNs also use batch normalization and dropout for regularization.\nfor i in range(15):    \n    cnn = keras.Sequential()\n\n    cnn.add(layers.Conv2D(filters=32, kernel_size=3, activation='relu', input_shape=(28, 28, 1)))\n    cnn.add(layers.BatchNormalization())\n    \n    cnn.add(layers.Conv2D(filters=32, kernel_size=3, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    \n    cnn.add(layers.Conv2D(filters=32, kernel_size=5, strides=2, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    cnn.add(layers.Dropout(0.25))\n\n    cnn.add(layers.Conv2D(filters=64, kernel_size=3, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    \n    cnn.add(layers.Conv2D(filters=64, kernel_size=3, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    \n    cnn.add(layers.Conv2D(filters=64, kernel_size=5, strides=2, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    cnn.add(layers.Dropout(0.25))\n\n    cnn.add(layers.Flatten())\n    cnn.add(layers.Dense(256, activation='relu'))\n    cnn.add(layers.BatchNormalization())\n    cnn.add(layers.Dropout(0.5))\n\n    cnn.add(layers.Dense(10, activation='softmax'))\n    \n    ensemble.append(cnn)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"steps = np.ceil(len(X) / 32) # Calculate the number of steps in an epoch (using batches of size 32)\n\n# Train the CNNs (this might take several hours)\nfor i in range(15):\n    \n    # Exponential decay schedule for the learning rate. Values for initial_learning_rate and decay_rate were found using grid search.\n    opt = keras.optimizers.Adam(learning_rate=ExponentialDecay(initial_learning_rate=0.003, decay_steps=steps, decay_rate=0.9))\n\n    ensemble[i].compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n\n    ensemble[i].fit(idg.flow(X, yh), epochs=30, verbose=0)\n    \n    print(str(i+1) + 'th NN trained')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"probs = np.zeros((28000, 10))\n# Add up predictions (probabilities) from all the CNNs\nfor i in range(15):\n    probs += ensemble[i].predict(test.reshape(28000, 28, 28, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat = np.argmax(probs, axis=1) # Make the predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the submission output\nsub = pd.DataFrame(y_hat, index=list(range(1, 28001)), columns=['Label'])\nsub.index.name = 'ImageId'\nsub.to_csv('sub.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}