{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# I/O\ntrain_file = '../input/digit-recognizer/train.csv'\ntest_file = '/kaggle/input/digit-recognizer/test.csv'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# for plotting\nimport matplotlib.pyplot as plt  \nimport seaborn as sns\n\n# for data augmentation\nfrom sklearn.model_selection import train_test_split  \nfrom keras.preprocessing.image import ImageDataGenerator\n\n# preprocessing\nfrom keras.utils import to_categorical\n\n# neural net\nfrom keras.utils import plot_model\nfrom keras.models import Sequential, Model, load_model\nfrom keras.layers import Input, Activation, Dense, Dropout, Flatten, concatenate\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.callbacks import LearningRateScheduler, Callback\nfrom IPython.display import clear_output\n\n# Seed for reproducibility\nseed = 42\nnp.random.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Data preprocessing"},{"metadata":{},"cell_type":"markdown","source":"First we we load available data."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(train_file)\ntest = pd.read_csv(test_file)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's have a look at train data."},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We see labels for digits (0 to 9), that are attached to vectors (1, 784) of pixel values. Let's also see test data format. "},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.1. Checking missing values"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking missing values\ndisplay(train.isnull().any().sum())\ndisplay(test.isnull().any().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I will also check the distribution of training data"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.barplot(train['label'].value_counts().index, train['label'].value_counts().values)\nplt.xlabel('Label', fontsize=14)\nplt.ylabel('# of samples', fontsize=14);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.2 Features/labels split"},{"metadata":{},"cell_type":"markdown","source":"Now I will split training data into features and labels. "},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.iloc[:, 1:]  # features\ny = train.iloc[:, 0]  # labels","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.3. Reshape"},{"metadata":{},"cell_type":"markdown","source":"Reshape training and test data into images (28x28 pixels) and matrixes suitable for CNN and convolution procedure in particular (type = np.ndarray). \\\nOur matrices take shape (sample_number, x, y, channels). We have grayscale images, therefore we have only one channel."},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X.values.reshape(-1, 28, 28, 1)\ntest_arr = test.values.reshape(-1, 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Let's take a look at our random training sample"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = X[np.random.randint(0, len(X)), :, :, 0]\nfig, ax = plt.subplots(figsize=(12,12))\nax.imshow(sample, cmap='binary')\nfor (j,i),label in np.ndenumerate(sample):\n    ax.text(i,j,label,ha='center',va='center');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As neural nets need normalized data, I normalize arrays using mean normalization."},{"metadata":{},"cell_type":"markdown","source":"## 1.4. Normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Mean normalization function\ndef normalize(arr): \n    return (arr - np.mean(arr)) / np.std(arr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Normalizing training and test data \nX = normalize(X)\ntest = normalize(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sanity check\nmean, std = np.mean(X), np.std(X)\nprint('Mean: %.3f, Standard Deviation: %.3f' % (mean, std))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.5. Image augmentation\nI will import ImageDataGenerator class and configure it to my application. Be aware, that heavy augmentation can cause training and validation curves to inverse during training. "},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n            rotation_range=20,  # randomly rotate images in the range (degrees, 0 to 20)\n            zoom_range = 0.1, # Randomly zoom image \n            width_shift_range=0.2,  # randomly shift images horizontally (fraction of total width)\n            height_shift_range=0.2,  # randomly shift images vertically (fraction of total height)\n            horizontal_flip=False, \n            vertical_flip=False)  ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will augment only training data, therefore I will use sklearn's train_test_split function to divide my training dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next we will fit data generator"},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.6. OneHotEncoding for labels"},{"metadata":{},"cell_type":"markdown","source":"We will use OneHotEncoding on our labels, as Neural Nets output results in vectors of shape (n_labels, 1)."},{"metadata":{"trusted":true},"cell_type":"code","source":"# OHE of the output - encoding 10 different digits \n\ny_train = to_categorical(y_train, num_classes=10)\ny_test = to_categorical(y_test, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Building Convolutional Neural Network with Inception Layer\nI will use Keras functional API to build CNN with Inception Block, as it allows to create multicommected layers. First, we will define our Convolutional + MaxPool Block. "},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_bn_pool_block(X, filters, kernel, pool_strides):\n        \n        X = Conv2D(filters=filters, kernel_size=kernel, activation='relu', strides=1, padding='same', data_format='channels_last')(X)\n        X = BatchNormalization()(X)\n        X = MaxPooling2D(pool_size=(2, 2), strides=pool_strides, padding='valid')(X)\n\n        return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def inc_block_A(X):\n    block_1 = Conv2D(96, (1,1), padding='same', activation='relu')(X)\n    block_1 = Conv2D(96, (3,3), padding='same', activation='relu')(block_1)\n    block_2 = Conv2D(64, (1,1), padding='same', activation='relu')(X)\n    block_2 = Conv2D(96, (3,3), padding='same', activation='relu')(block_2)\n    block_2 = Conv2D(96, (3,3), padding='same', activation='relu')(block_2)\n    block_3 = MaxPooling2D((2,2), strides=1, padding='same')(X)\n    block_3 = Conv2D(96, (1,1), padding='same', activation='relu')(block_3)\n    block_4 = Conv2D(96, (1,1), padding='same', activation='relu')(X)\n    \n    output = concatenate([block_1, block_2, block_3, block_4], axis = 3)\n    return output","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next, we will build out CNN. "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Buildng CNN\ndef build_model():\n    \n    # Define input\n    input_img = Input(shape=(28,28,1))\n    \n    # Building stem\n    stem = Conv2D(32, (3,3), padding='same', activation='relu')(input_img)\n    stem = BatchNormalization()(stem)\n    stem = Conv2D(48, (3,3), padding='same', activation='relu')(stem)\n    stem = BatchNormalization()(stem)\n    stem = Conv2D(64, (3,3), padding='same', activation='relu')(stem)\n    stem = BatchNormalization()(stem)\n    stem = Dropout(0.2)(stem)\n    \n    # Inception block\n    incept = inc_block_A(stem)\n    X = Dropout(0.1)(incept)\n    X = inc_block_A(X)\n    X = Dropout(0.1)(X)\n    X = inc_block_A(X)\n    X = Dropout(0.1)(X)\n    X = inc_block_A(X)\n    X = Dropout(0.1)(X)\n    X = MaxPooling2D((2,2), strides=2, padding='same')(X)\n    \n    X = Conv2D(64, (3,3), strides=1, padding='same', activation='relu')(X)\n    X = MaxPooling2D((2,2), strides=2, padding='same')(X)\n    X = Conv2D(64, (3,3), strides=2, padding='same', activation='relu')(X)\n    X = MaxPooling2D((2,2), strides=2, padding='same')(X)\n\n    \n    # Adding fully-conected layers with drop-out for regularization\n    X = Flatten()(X)\n    X = Dense(1024, activation='relu')(X)\n    X = BatchNormalization()(X)\n    X = Dropout(0.4)(X)\n    X = Dense(512, activation='relu')(X)\n    X = BatchNormalization()(X)\n    X = Dropout(0.2)(X)\n    \n    # Adding output softmax layer for multilabel classification\n    output = Dense(10, activation='softmax')(X)\n    \n    model = Model(inputs=input_img, outputs=output, name=\"mnist_model\")\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mnist_model = build_model()\nmnist_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"plot_model(mnist_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Adding optimizer\noptimizer = Adam(lr=0.0007, beta_1=0.9, beta_2=0.999) # beta-1 and beta-2 are \"no brainers\" - Andrew Ng ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Including learning rate decay - so to prevent overfitting\nreduce_lr = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Live plotting the loss function\nclass PlotLosses(Callback):\n    def on_train_begin(self, logs={}):\n        self.i = 0\n        self.x = []\n        self.losses = []\n        self.val_losses = []\n        self.accuracy = []\n        self.val_accuracy = []\n        \n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        \n        self.logs.append(logs)\n        self.x.append(self.i)\n        self.losses.append(logs.get('loss'))\n        self.val_losses.append(logs.get('val_loss'))\n        self.accuracy.append(logs.get('categorical_accuracy'))\n        self.val_accuracy.append(logs.get('val_categorical_accuracy'))\n        self.i += 1\n        \n        clear_output(wait=True)\n        fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12,5))\n        ax1.plot(self.x, self.losses, label=\"loss\")\n        ax1.plot(self.x, self.val_losses, label=\"val_loss\")\n        ax1.legend()\n        ax2.plot(self.x, self.accuracy, label=\"accuracy\")\n        ax2.plot(self.x, self.val_accuracy, label=\"val_accuracy\")\n        ax2.legend()\n        plt.show();\n        \nplot_losses = PlotLosses()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Defining model parameters \nbatch_size = 64\nepochs = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compiling model\nmnist_model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['categorical_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fitting the model\nhistory = mnist_model.fit_generator(datagen.flow(X_train, y_train, batch_size = batch_size), \n                                    epochs = epochs,\n                                    validation_data = (X_test, y_test),\n                                    verbose=0, \n                                    steps_per_epoch=X_train.shape[0] // batch_size,\n                                    callbacks = [reduce_lr, plot_losses])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Diagnostics\nplt.plot(history.history['loss'], label='train')\nplt.plot(history.history['val_loss'], label='test')\nplt.title(\"Model Loss\")\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(history.history['categorical_accuracy'], label='train')\nplt.plot(history.history['val_categorical_accuracy'], label='test')\nplt.title('Model Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predicting for the test set \npreds = np.argmax(mnist_model.predict(test_arr), axis=1)\n    \nsub_df = {'ImageId':list(range(1, len(test) + 1)),'Label':preds}\nsubmission = pd.DataFrame(sub_df).astype('int')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.barplot(x=submission['Label'].value_counts().index, y=submission['Label'].value_counts().values);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}