{"cells":[{"metadata":{},"cell_type":"markdown","source":"1. Introduction\n2. Data preparation\n3. Data augmentation\n4. Maxout network\n5. Evaluation\n6. Submission"},{"metadata":{},"cell_type":"markdown","source":"# 1. Introduction"},{"metadata":{},"cell_type":"markdown","source":"The aim of this notebook is to test the effectiveness of Maxout networks as described in [Goodfellow et al. (2013)](https://arxiv.org/pdf/1302.4389.pdf).\n\nCheck [here](https://www.kaggle.com/schateau/cnn-with-ic-layer) for a notebook that uses Independent-Component layer (i.e. BatchNorm + Dropout) as described in [Chen et al. (2019)](https://arxiv.org/abs/1905.05928)."},{"metadata":{},"cell_type":"markdown","source":"# 2. Data preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Basic CNN for classifying dogs and cats pictures\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport random\nimport math\n\n# Input data files are available in the \"../input/\" directory.\nimport os\nprint(os.listdir(\"../input/dogs-vs-cats\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Prepare training data\nfilenames = os.listdir(\"../input/dogs-vs-cats/train/train\")\ncategories = []\nfor filename in filenames:\n    category = filename.split('.')[0]\n    if category == 'dog':\n        categories.append(1)\n    else:\n        categories.append(0)\n\nprint('1=dog; 0=cat')\ndf = pd.DataFrame({\n    'filename': filenames,\n    'category': categories\n})\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#How are the pictures distributed ?\ndf['category'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"427300c6-b035-4c35-9573-abae8d379f74","_cell_guid":"208f79d7-15b9-4a96-b710-c938dd81ba7d","trusted":true},"cell_type":"code","source":"#See a random sample:\nfrom keras.preprocessing.image import load_img\n\nsample = random.choice(filenames)\nimage = load_img(\"../input/dogs-vs-cats/train/train/\"+sample)\nplt.imshow(image)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#define image shape:\nheight = 150\nwidth = 150\nchannels = 3\nimage_shape = (height, width, channels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Because we'll use a generator with binary classification for the training set, we must pass from 'int' to 'string' for the y_col=\"category\" column\ndf[\"category\"] = df[\"category\"].replace({0: 'cat', 1: 'dog'}) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#splitting into training and validation sets:\nfrom sklearn.model_selection import train_test_split\n\n#validation set:\ntrain_df_tmp, validate_df = train_test_split(df, test_size=0.1, random_state=2)\n#validation set:\ntrain_df, test_df = train_test_split(train_df_tmp, test_size=0.1, random_state=2)\n\n#print the number of samples in each set:\nprint('Number of samples in train_df:', len(train_df), \n      '\\nNumber of samples in validate_df:', len(validate_df),\n      '\\nNumber of samples in validate_df:', len(test_df)\n     )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Reduce the sizes of the training and validation sets when testing code to save some GPU time.\n#validate_df = validate_df.sample(n=1000).reset_index() # use for fast testing code purpose\n#train_df = train_df.sample(n=5000).reset_index() # use for fast testing code purpose\n\n#For training on the full set, comment the lines above and uncomment the ones that follows:\ntrain_df = train_df.reset_index()\nvalidate_df = validate_df.reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\ntotal_train = train_df.shape[0]\ntotal_validate = validate_df.shape[0]\nepochs = 100","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3. Data augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Data preprocessing:\nfrom keras.preprocessing.image import ImageDataGenerator\n\n#ImageDataGenerator with data augmentation:\ntrain_datagen = ImageDataGenerator(\n    rescale=1./255,\n    rotation_range=10,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n    shear_range=0.1,\n    zoom_range=0.2,\n    horizontal_flip=True,\n    fill_mode='nearest')\n#Note: We choose \"small\" value of the parapeters for Data augmentation.\n#Indeed, Data augmentation is done only on the training set, and thus might lead to a validation set containing \"easier\" cases to predict.\n\n#Generator for training:\ntrain_generator = train_datagen.flow_from_dataframe(\n    dataframe=train_df,\n    directory='../input/dogs-vs-cats/train/train/', \n    x_col='filename',\n    y_col='category',\n    class_mode='binary',\n    target_size=(height, width),\n    batch_size=batch_size\n)\n\n#The images outputed by the validation generator should not be augmented:\nvalidation_datagen = ImageDataGenerator(rescale=1./255)\n\nvalidation_generator = validation_datagen.flow_from_dataframe(\n    dataframe=validate_df, \n    directory='../input/dogs-vs-cats/train/train/',\n    x_col='filename',\n    y_col='category',\n    class_mode='binary',\n    target_size=(height, width),\n    batch_size=batch_size\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Displaying some randomly augmented training images\nexample_df = train_df.sample(n=1).reset_index(drop=True)\nexample_generator = train_datagen.flow_from_dataframe(\n    dataframe=example_df,\n    directory='../input/dogs-vs-cats/train/train/', \n    x_col='filename',\n    y_col='category',\n    class_mode='categorical'\n)\nplt.figure(figsize=(12, 12))\nfor i in range(0, 9):\n    plt.subplot(3, 3, i+1)\n    for X_batch, Y_batch in example_generator:\n        image = X_batch[0]\n        plt.imshow(image)\n        break\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 4. Maxout network"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\n#Definition of the Maxout layer:\ndef Maxout(inputs, num_units, axis=None):\n    \"\"\"\n    Maxout OP as in the paper https://arxiv.org/pdf/1302.4389.pdf\n    \n    Max pooling is performed on the filter/channel dimension. This can also be\n    used after fully-connected layers to reduce number of features.\n\n    Args:\n        inputs (Tensor): A NHWC (or NCHW) tensor on which maxout will be performed. The number of channels C has to be known.\n        num_units (int): Specifies how many features will remain after the max pooling operation performed in the filter/channel dimension. num_unit must be a multiple of num_channels, so that num_channels=num_unit*K where K stands for the number of layers (in the channel dimension) where the Max pooling operation is done. Typically, K=2, 3, or 4\n        axis: The dimension where max pooling will be performed. Default is the last dimension.\n        \n    Returns:\n        A 'Tensor' of shape (B,H,W,num_units) named ``output``.\n    \n    Raises:\n        ValueError: if num_units is not multiple of number of features.\n\"\"\"\n    \n    input_shape = inputs.get_shape().as_list()\n    ndim = len(input_shape)\n    assert ndim == 4\n    \n    if axis is None:  # Assume that channel is the last dimension\n        axis = -1\n    num_channels = input_shape[axis]\n    assert num_channels is not None \n    \n    if num_channels % num_units:\n        raise ValueError('number of features({}) is not a multiple of num_units({})'.format(num_channels, num_units))\n    \n    input_shape[axis] = num_units\n    input_shape+= [num_channels // num_units]\n    \n    outputs = K.reshape(inputs, (-1, input_shape[1], input_shape[2], input_shape[3], input_shape[4]))\n    outputs = K.max(outputs, axis=-1, keepdims=False)\n\n    return outputs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\n#Testing function Maxout\nif __name__ == '__main__':\n    with tf.Session() as sess:\n        x = tf.Variable(np.random.uniform(size=(1, 25, 10, 500)))\n        y = tf.square(x)\n        mo = Maxout(x, 50, axis=None)\n        sess.run(tf.global_variables_initializer())\n\n        print(mo.eval().shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Building a Maxout network for efficient Dropout\nfrom keras.models import Model\nfrom keras import Input, optimizers\nfrom keras.layers import Dropout, Lambda, Conv2D, MaxPooling2D, Flatten, Dense\nfrom keras.constraints import max_norm\n\n# To adapt the network with Dropout, we'll follow these steps:\n# 1. use maxout instead of RelU (https://arxiv.org/pdf/1302.4389.pdf)\n# 2. setting a maxnorm constraint on the weights \n# 3. use Stochastic Gradient Descent with high scheduled decaying learning rate, and large momentum\n\nmax_norm4 = max_norm(max_value=4, axis=[0, 1, 2])\n\ninput_tensor = Input(shape=image_shape)\n\n#We use a week value (p=0.1) for the dropout units that act before Conv2D layers. Two reasons for that: \n# 1) Conv2D layers are not the more prone to overfitting, and \n# 2) it will provide some noisy inputs for the higher fully connected layers.\nx = Conv2D(32, (3, 3), padding='same', kernel_constraint=max_norm4)(input_tensor)\nx = Lambda(Maxout, arguments={'num_units':16}, name='act1')(x)\nx = MaxPooling2D((2, 2))(x)\nx = Dropout(0.1)(x)\n\nx = Conv2D(64, (3, 3), padding='same', kernel_constraint=max_norm4)(x)\nx = Lambda(Maxout, arguments={'num_units':32}, name='act2')(x)\nx = MaxPooling2D((2, 2))(x)\nx = Dropout(0.1)(x)\n\nx = Conv2D(128, (3, 3), padding='same', kernel_constraint=max_norm4)(x)\nx = Lambda(Maxout, arguments={'num_units':64}, name='act3')(x)\nx = MaxPooling2D((2, 2))(x)\nx = Dropout(0.1)(x)\n\nx = Conv2D(128, (3, 3), padding='same', kernel_constraint=max_norm4)(x)\nx = Lambda(Maxout, arguments={'num_units':64}, name='act4')(x)\nx = MaxPooling2D((2, 2))(x)\nx = Dropout(0.1)(x)\n\nx = Flatten()(x)\n\nx = Dense(512, activation='relu', kernel_constraint=max_norm(4))(x)\nx = Dropout(0.5)(x) #We use p=0.5 for this dropout unit to fight overfitting\noutput_tensor = Dense(1, activation=\"sigmoid\")(x)\n\nMaxout_model = Model(input_tensor, output_tensor)\nmodel_name = Maxout_model.name\n\n#Dropout is most effective when taking relatively large steps in parameter space: lr=0.01\n#Note: be careful with the momentum value: indeed, at the beginning, the optimizer may go in same direction as the gradient (which is good) some long time. \n#However, this may cause a very big momentum if the \"momentum\" parameter is set too high.\n#It can results in \"climbing hills\" with the optimizer, and thus a possible increase of the training loss.\nsgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) \n\nMaxout_model.compile(loss='binary_crossentropy',\n                     optimizer=sgd,\n                     metrics=['acc'])\n\nMaxout_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, LearningRateScheduler\n\n#def step_decay(epoch):\n#    initial_lrate = 0.01\n#    drop = 0.1\n#    epochs_drop = 40.0\n#    lrate = initial_lrate * math.pow(drop,math.floor((1+epoch)/epochs_drop))\n#    return lrate\n#lrate = LearningRateScheduler(step_decay)\n\n#filepath = 'best_{0}'.format(model_name) + '-{epoch:03d}-{val_acc:03f}.h5'\n#mcp = ModelCheckpoint(filepath, monitor='val_loss', mode='min', save_best_only=True, verbose=1)\nearlystop = EarlyStopping(monitor='val_loss',\n                          mode='min',\n                          patience=20,\n                          verbose=0)\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=10, \n                                            verbose=1, \n                                            factor=0.1, \n                                            min_lr=1e-5)\ncallbacks = [earlystop, learning_rate_reduction]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fitting the model using a batch generator:\nhistory = Maxout_model.fit_generator(\n    train_generator,\n    steps_per_epoch=total_train//batch_size,\n    epochs=epochs,\n    validation_data=validation_generator,\n    validation_steps=total_validate//batch_size,\n    callbacks=callbacks,\n    verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define a smooth function to display the training and validation curves\ndef plot_smoothed_learning_curves(history):\n    val_loss = history.history['val_loss']#[-30:-1] #Uncomment if you want to see only the last epochs\n    loss = history.history['loss']#[-30:-1]\n    acc = history.history['acc']#[-30:-1]\n    val_acc = history.history['val_acc']#[-30:-1]\n    \n    epochs = range(1, len(acc)+1 )\n    \n    # Plot the loss and accuracy curves for training and validation \n    fig, ax = plt.subplots(2,1, figsize=(12, 12))\n    ax[0].plot(epochs, smooth_curve(loss), 'bo', label=\"Smoothed training loss\")\n    ax[0].plot(epochs, smooth_curve(val_loss), 'b', label=\"Smoothed validation loss\",axes =ax[0])\n    legend = ax[0].legend(loc='best', shadow=True)\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Loss')\n\n    ax[1].plot(epochs, smooth_curve(acc), 'bo', label=\"Smoothed training accuracy\")\n    ax[1].plot(epochs, smooth_curve(val_acc), 'b',label=\"Smoothed validation accuracy\")\n    legend = ax[1].legend(loc='best', shadow=True)\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel('Accuracy')\n    return\n\ndef smooth_curve(points, factor=0.8):\n    smoothed_points = []\n    for point in points:\n        if smoothed_points:\n            previous = smoothed_points[-1]\n            smoothed_points.append(previous*factor + point*(1-factor))\n        else:\n            smoothed_points.append(point)\n    return smoothed_points","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Visualisation:\nplot_smoothed_learning_curves(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The network has converged. Let's evaluate it now."},{"metadata":{},"cell_type":"markdown","source":"## 4.5 Evaluation of the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define test_generator:\ntest_datagen = ImageDataGenerator(rescale=1./255)\ntest_generator = test_datagen.flow_from_dataframe(\n    dataframe = test_df, \n    directory=\"../input/dogs-vs-cats/train/train/\", \n    x_col='filename',\n    y_col='category',\n    class_mode='binary',\n    target_size=(height, width),\n    batch_size=batch_size,\n    shuffle=False\n)\n\nnb_samples = test_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Evaluate the model:\ntest_loss, test_acc = Maxout_model.evaluate_generator(test_generator, steps=np.ceil(nb_samples/batch_size))\nprint('test acc:', test_acc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"raw","source":"We reach an accuracy around 94~95%. Not bad for a model without pre-training."},{"metadata":{},"cell_type":"markdown","source":"## Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load test data:\ntest_filenames = os.listdir(\"../input/dogs-vs-cats/test1/test1\")\nreal_test_df = pd.DataFrame({\n    'filename': test_filenames\n})\n\nreal_test_df = real_test_df\nnb_samples = real_test_df.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Generator:\nreal_test_generator = test_datagen.flow_from_dataframe(\n    dataframe = real_test_df, \n    directory=\"../input/dogs-vs-cats/test1/test1/\", \n    x_col='filename',\n    y_col=None,\n    class_mode=None,\n    target_size=(height, width),\n    batch_size=batch_size,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Prediction:\npredict = Maxout_model.predict_generator(real_test_generator, steps=np.ceil(nb_samples/batch_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real_test_df['category'] = predict.round().astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"real_test_df['category'].value_counts().plot.bar()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = real_test_df.copy()\nsubmission_df['id'] = submission_df['filename'].str.split('.').str[0]\nsubmission_df['label'] = submission_df['category']\nsubmission_df.drop(['filename', 'category'], axis=1, inplace=True)\nsubmission_df.to_csv('submission_MaxOut.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}