{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Task\nhttps://www.kaggle.com/c/Kannada-MNIST"},{"metadata":{},"cell_type":"markdown","source":"# Load dependencies and data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom scipy import stats\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import MinMaxScaler, FunctionTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n\nimport keras\nfrom keras.models import Model, Sequential, load_model\nfrom keras.layers import Input, Dense, Conv2D, MaxPooling2D, GlobalMaxPool2D\nfrom keras.layers import Activation, Add, ReLU, Flatten, Dropout, BatchNormalization\nfrom keras.layers import ZeroPadding2D, Concatenate\nfrom keras.layers.advanced_activations import LeakyReLU\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.optimizers import Adam\nfrom keras import backend as K\nfrom keras.utils import to_categorical\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nfrom keras.applications import ResNet50\n\n\nplt.rcParams['figure.figsize'] = (12, 8)\nplt.rcParams['image.interpolation'] = 'nearest'\nplt.rcParams['image.cmap'] = 'gray'\nK.set_image_data_format('channels_last')\nrandom_state = 42\nimage_shape = (28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"source_df = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntest_df = pd.read_csv('../input/Kannada-MNIST/test.csv')\ndig_df = pd.read_csv('../input/Kannada-MNIST/Dig-MNIST.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_X_Y(df):\n    X = df.loc[:, 'pixel0':'pixel783'].copy()\n    Y = df.loc[:, 'label'].copy()\n    return (X, Y)\n\nsource_X, source_Y = get_X_Y(source_df) \ndig_X, dig_Y = get_X_Y(dig_df)\n\nprint('source_X shape {}, source_Y shape {}'.format(source_X.shape, source_Y.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_classes = sorted(source_Y.unique())\nnum_classes = len(Y_classes)\nplt.figure()\n\nfor y_idx, y_class in enumerate(Y_classes):\n    idxs = np.random.choice(np.c_[source_Y.loc[source_Y == y_class].index].flatten(), 10, replace=False)\n    for i, idx in enumerate(idxs):\n        plt_idx = i * num_classes + y_idx + 1\n        plt.subplot(num_classes, num_classes, plt_idx)\n        plt.imshow(source_X.iloc[idx].values.reshape(28, 28))\n        plt.axis('off')\n        if i == 0:\n            plt.title(str(y_class))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_X, valid_X, train_Y, valid_Y = train_test_split(\n    source_X, source_Y, test_size=0.2, stratify=source_Y, random_state=random_state)\nprint('train_X shape {}, train_Y shape {}, valid_X shape {}, valid_Y shape {}'.format(\n    train_X.shape, train_Y.shape, valid_X.shape, valid_Y.shape))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_cnn(input_shape, classes):\n    input_layer = Input(shape=input_shape)\n\n    conv_layer1 = Conv2D(32, kernel_size=(3,3), strides=1, padding='same', input_shape=(28, 28, 1))(input_layer)    \n    conv_layer1 = Conv2D(16, kernel_size=(3,3), strides=1, padding='same')(conv_layer1)\n    batch_norm_layer1 = BatchNormalization(momentum=0.5, gamma_initializer='uniform')(conv_layer1)\n    relu_layer1 = LeakyReLU()(batch_norm_layer1)\n    \n    max_pool_layer1 = MaxPooling2D(pool_size=(2,2), strides=(2,2))(relu_layer1)\n    con_drop_layer1 = Dropout(0.5)(max_pool_layer1)\n    \n    conv_layer2 = Conv2D(64, kernel_size=(3,3), strides=1, padding='same')(con_drop_layer1)    \n    conv_layer2 = Conv2D(32, kernel_size=(3,3), strides=1, padding='same')(conv_layer2)\n    batch_norm_layer2 = BatchNormalization(momentum=0.5, gamma_initializer='uniform')(conv_layer2)\n    relu_layer2 = LeakyReLU()(batch_norm_layer2)\n        \n    max_pool_layer2 = MaxPooling2D(pool_size=(2,2), strides=(2,2))(relu_layer2)\n    con_drop_layer2 = Dropout(0.5)(max_pool_layer2)\n    \n    conv_layer3 = Conv2D(128, kernel_size=(3,3), strides=1, padding='same')(con_drop_layer2)\n    conv_layer3 = Conv2D(64, kernel_size=(3,3), strides=1, padding='same')(conv_layer3)\n    batch_norm_layer3 = BatchNormalization(momentum=0.5, gamma_initializer='uniform')(conv_layer3)\n    relu_layer3 = LeakyReLU()(batch_norm_layer3)\n    \n    max_pool_layer3 = MaxPooling2D(pool_size=(2,2), strides=(2,2))(relu_layer3)\n    flatten = Flatten()(max_pool_layer3)  \n    dropout_layer1 = Dropout(0.5)(flatten)\n    \n    dense_layer1 = Dense(256)(dropout_layer1)\n    relu_layer4 = LeakyReLU()(dense_layer1)\n    dropout_layer2 = Dropout(0.5)(relu_layer4)\n    \n    dense_layer2 = Dense(256)(dropout_layer2)\n    relu_layer5 = LeakyReLU()(dense_layer2)\n    dropout_layer3 = Dropout(0.5)(relu_layer5)\n    \n    output_layer = Dense(10, activation='softmax')(dropout_layer3)\n\n    model = Model(inputs=[input_layer], outputs=[output_layer])\n    model.compile('adam', loss='categorical_crossentropy', metrics=['accuracy'])\n    \n    return model\n\ncnn_model = build_cnn(image_shape, 10)\ncnn_model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Util functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preproc_images(images):\n    return (images / 255.0).astype(np.float64).values.reshape((-1, 28, 28, 1))\n    \ndef agg_result_mode(predictions):\n    mode_result = stats.mode(predictions.argmax(axis=2), axis=0)\n    result = mode_result.mode.squeeze()    \n    return result\n\ndef agg_result_mean(predictions):\n    return predictions.mean(axis=0).argmax(axis=1)\n\ndef show_class_reports(y_true, y_predicted):\n    print('accuracy {}'.format(accuracy_score(y_true, y_predicted)))\n    print(classification_report(y_true, y_predicted))\n    print(confusion_matrix(y_true, y_predicted))\n\ndef plot_graphs(history, string):\n    plt.plot(history.history[string])\n    plt.plot(history.history['val_'+string], '')\n    plt.xlabel(\"Epochs\")\n    plt.ylabel(string)\n    plt.legend([string, 'val_'+string])\n    plt.show()    \n\ndef get_image_data_gen():\n    return ImageDataGenerator(\n        rotation_range=15,\n        zoom_range=0.15,\n        shear_range=0.15,\n        width_shift_range=0.15,\n        height_shift_range=0.15\n    )\n\ndef submit(model, kfold=False):\n    test_ID = test_df.loc[:, 'id']\n    test_X = test_df.loc[:, 'pixel0':'pixel783']\n    test_X = preproc_images(test_X)\n    \n    if kfold:\n        test_Ys = np.array([m.predict(test_X) for m in model])\n        test_Y = agg_result_mean(test_Ys)    \n    else:\n        test_Y = model.predict(test_X)\n        test_Y = np.argmax(test_Y, axis=1)\n\n    submition_df = pd.DataFrame.from_dict({ 'id': test_ID, 'label': test_Y })\n    submition_df.to_csv('./submission.csv', index=False)\n\n\ndef run_model(model, X, y, batch_size, number=0, **kwards):\n    best_model_path = 'best_model_{}.h5'.format(number)\n    \n    image_data_gen = get_image_data_gen()\n    image_data_gen.fit(X)\n    \n    callbacks = [\n        EarlyStopping(monitor='val_accuracy', min_delta=0.001, patience=15, verbose=1, mode='auto'),\n        ModelCheckpoint(best_model_path, monitor='val_accuracy', verbose=1, mode='auto', save_best_only=True),\n        ReduceLROnPlateau(monitor='val_accuracy', factor=0.7, patience=2, verbose=1, mode='auto')\n    ]\n    \n    images_flow = image_data_gen.flow(X, y, batch_size=batch_size)  \n    steps_per_epoch = X.shape[0] // batch_size\n    hist = model.fit(images_flow, steps_per_epoch=steps_per_epoch, callbacks=callbacks, **kwards)\n    model.load_weights(best_model_path)\n\n    return hist","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_count = 6\nepochs = 25\nbatch_size = 128\n\nmodels = [build_cnn(image_shape, num_classes) for _ in range(model_count)]\nkfold_splitter = StratifiedKFold(n_splits=model_count, shuffle=True, random_state=random_state)\nhistories = []\n\nfor index, (train_index, valid_index) in enumerate(kfold_splitter.split(source_X, source_Y)):\n    print('Start model {} training'.format(index + 1))\n    \n    k_fold_train_X, k_fold_train_Y = source_X.iloc[train_index], source_Y.iloc[train_index]\n    k_fold_valid_X, k_fold_valid_Y = source_X.iloc[valid_index], source_Y.iloc[valid_index]\n    \n    k_fold_train_X_prep, k_fold_valid_X_prep = preproc_images(k_fold_train_X), preproc_images(k_fold_valid_X)\n    \n    run_model(\n        models[index], k_fold_train_X_prep, to_categorical(k_fold_train_Y), batch_size, number=index,\n        epochs=epochs, validation_data=(k_fold_valid_X_prep, to_categorical(k_fold_valid_Y))\n    )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for hist in histories:\n    plot_graphs(hist, 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"submit(models, kfold=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}