{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\nsub = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')\nprint('Our train set have {} rows and {} columns'.format(train.shape[0], train.shape[1]))\nprint('Our test set have {} rows and {} columns'.format(test.shape[0], test.shape[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# target variable distribution\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.figure(figsize = (10,8))\nsns.countplot(x = 'label', data = train)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have 6000 examples for each label in the training set"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No missing values. Just checking :)."},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n\nWe want to reshape both dataframes (train, test) to adjust the dimensions that our CNN is going to take as input. This are grey scale images so the channel is going to be 1 (channel is the last dimension, for color images we have a channel of 3 (RGB)."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndef preprocessing(train, test):\n    # drop label column of the train set and reshape, in this case we have 28X28 pixel images\n    IMG_SIZE = 28\n    img_train = train.drop(['label'], axis = 1).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1).astype('float32')\n    img_test = test.drop(['id'], axis = 1).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1).astype('float32')\n    img_y = train['label'].values\n    # scale data (rgb goes from 0 to 255, dividing by 255 change the range to 0-1)\n    img_train /= 255\n    img_test /= 255\n    # taking 20% of our train data as eval data.\n    x_train, x_val, y_train, y_val = train_test_split(img_train, img_y, test_size = 0.20)\n    print('Our transformed train set have the following dimension: ', x_train.shape)\n    print('Our transformed valid set have the following dimension: ', x_val.shape)\n    return img_test, x_train, x_val, y_train, y_val","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras\nfrom keras.layers import Input, Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, Activation, MaxPooling2D, BatchNormalization\nfrom keras import backend as K\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import Model\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ReduceLROnPlateau\n# build model\nn_classes = train['label'].value_counts().count()\ndef build_model(input_shape=(28, 28, 1), classes = n_classes):\n    input_layer = Input(shape=input_shape)\n    x = Conv2D(16, (3,3), strides=1, padding=\"same\", name=\"conv1\")(input_layer)\n    x = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch1\")(x)\n    x = Activation('relu',name='relu1')(x)\n    x = Dropout(0.1)(x)\n    \n    x = Conv2D(32, (3,3), strides=1, padding=\"same\", name=\"conv2\")(x)\n    x = BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch2\")(x)\n    x = Activation('relu',name='relu2')(x)\n    x = Dropout(0.15)(x)\n    x = MaxPooling2D(pool_size=2, strides=2, padding=\"same\", name=\"max2\")(x)\n    \n    x = Conv2D(64, (5,5), strides=1, padding =\"same\", name=\"conv3\")(x)\n    x = BatchNormalization(momentum=0.17, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch3\")(x)\n    x = Activation('relu', name=\"relu3\")(x)\n    x = MaxPooling2D(pool_size=2, strides=2, padding=\"same\", name=\"max3\")(x)\n    \n    x = Conv2D(128, (5,5), strides=1, padding=\"same\", name=\"conv4\")(x)\n    x = BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch4\")(x)\n    x = Activation('relu', name=\"relu4\")(x)\n    x = Dropout(0.17)(x)\n    \n    x = Conv2D(64, (3,3), strides=1, padding=\"same\", name=\"conv5\")(x)\n    x = BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch5\")(x)\n    x = Activation('relu', name='relu5')(x)\n    x = Dropout(0.2)(x)\n    \n    x = Conv2D(32, (3,3), strides=1, padding=\"same\", name=\"conv6\")(x)\n    x = BatchNormalization(momentum=0.15, epsilon=1e-5, gamma_initializer=\"uniform\", name=\"batch6\" )(x)\n    \n    x = Activation('relu', name=\"relu6\")(x)\n    x = Dropout(0.05)(x)\n    \n    x = Flatten()(x)\n    x = Dense(50, name=\"Dense1\")(x)\n    x = Activation('relu', name='relu7')(x)\n    x = Dropout(0.05)(x)\n    x = Dense(25, name=\"Dense2\")(x)\n    x = Activation('relu', name='relu8')(x)\n    x = Dropout(0.03)(x)\n    x = Dense(classes, name=\"Dense3\")(x)\n    x = Activation('softmax')(x)\n\n    model = Model(inputs=input_layer, outputs=x)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# let's create a data generator to make some data augmentation\n# let's create a checkpoint callback to save best model in the training process\n# let's create a another callback to reduce the learning rate if the validation score dont improve in x round\ndef trng_lr_ck_opt(modelname):\n    train_generator = ImageDataGenerator(rotation_range = 8,  # we dont want to rotate that much (confuse)\n                                        zoom_range = 0.28,\n                                        width_shift_range = 0.25,\n                                        height_shift_range = 0.25)\n    learning_rate_reduction = ReduceLROnPlateau(monitor = 'val_accuracy', patience = 5, verbose = 1, factor = 0.5, min_le = 0.000001)\n    \n    checkpoint = ModelCheckpoint(modelname+'.hdf5', monitor = 'val_accuracy', verbose = 1, save_best_only = True)\n    return train_generator, learning_rate_reduction, checkpoint\n\ndef compile_model():\n    optimizer = Adam(lr = 0.001)\n    model = build_model(input_shape = (28, 28, 1), classes = n_classes)\n    model.compile(loss = 'sparse_categorical_crossentropy', optimizer = optimizer, metrics = ['accuracy'])\n    return model\n\n# final step, join all the modules and train the model\ndef train_and_evaluate(train, test, batch_size, epochs):\n    img_test, x_train, x_val, y_train, y_val = preprocessing(train, test)\n    model = compile_model()\n    train_generator, learning_rate_reduction, checkpoint = trng_lr_ck_opt('bestmodel')\n    history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size = batch_size),\n                                  steps_per_epoch = x_train.shape[0] // batch_size,\n                                  epochs = epochs, \n                                  validation_data = (x_val, y_val),\n                                  callbacks = [checkpoint, learning_rate_reduction])\n    return img_test, history\n\n# run train and evaluate\nBATCH_SIZE = 64\nEPOCHS = 40\nimg_test, history = train_and_evaluate(train, test, BATCH_SIZE, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_loss_acc(his, epoch):\n    fig, (ax1, ax2) = plt.subplots(2, 1, figsize = (15,10))\n    ax1.plot(np.arange(0, epoch), his.history['loss'], label = 'train_loss')\n    ax1.plot(np.arange(0, epoch), his.history['val_loss'], label = 'val_loss')\n    ax1.set_title('Loss')\n    ax1.figure.legend()\n    ax2.plot(np.arange(0, epoch), his.history['accuracy'], label = 'train_acc')\n    ax2.plot(np.arange(0, epoch), his.history['val_accuracy'], label = 'val_acc')\n    ax2.set_title('Accuracy')\n    ax2.figure.legend()\n    plt.show()\nplot_loss_acc(history, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Our val loss is better than our training loss. Cross validation is not a bad idea to try. Let's give it a shot.\n\nThis will evaluate all our training data, we are going to have a out of folds accuracy score."},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn import metrics\ndef train_and_evaluate_kfold(train, test, batch_size, epochs):\n    NFOLDS = 5\n    folds = KFold(n_splits=NFOLDS)\n    IMG_SIZE = 28\n    img_test = test.drop(['id'], axis = 1).values.reshape(-1, IMG_SIZE, IMG_SIZE, 1).astype('float32') / 255\n    img_y = train['label'].values\n    train = train.drop(['label'], axis = 1)\n    oof = np.zeros([train.shape[0], n_classes])\n    preds = np.zeros([test.shape[0], n_classes])\n    for fold_n, (train_index, val_index) in enumerate(folds.split(train)):\n        print('Training fold {}'.format(fold_n + 1))\n        x_train, x_val = train.iloc[train_index], train.iloc[val_index]\n        y_train, y_val = img_y[train_index], img_y[val_index]\n        x_train = x_train.values.reshape(-1, IMG_SIZE, IMG_SIZE, 1).astype('float32') / 255\n        x_val = x_val.values.reshape(-1, IMG_SIZE, IMG_SIZE, 1).astype('float32') / 255\n        model = compile_model()\n        train_generator, learning_rate_reduction, checkpoint = trng_lr_ck_opt('model_fold_{}'.format(fold_n + 1))\n        history = model.fit_generator(train_generator.flow(x_train, y_train, batch_size = batch_size),\n                                      steps_per_epoch = x_train.shape[0] // batch_size,\n                                      epochs = epochs, \n                                      validation_data = (x_val, y_val),\n                                      callbacks = [checkpoint, learning_rate_reduction])\n        model.load_weights('model_fold_{}'.format(fold_n + 1) + '.hdf5')\n        preds += model.predict(img_test) / NFOLDS\n        oof[val_index] = model.predict(x_val)\n        \n        \n    oof = np.argmax(oof, axis = 1)\n    preds = np.argmax(preds, axis = 1)\n    print('-'*20)\n    print('Out of fold cv score: ', metrics.accuracy_score(img_y, oof))\n    return preds, oof\n\n# let's train the models, load the best weight for each fold and predict the test and the training set(out of folds)\npreds, oof = train_and_evaluate_kfold(train, test, BATCH_SIZE, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# save predictions\nsub['label'] = preds\nsub.to_csv('cnn_5kfold_baseline.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}