{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nimport pandas as pd\nimport numpy as np\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nplt.style.use('ggplot')\n%matplotlib inline\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nimport tensorflow.keras.layers as l\nimport tensorflow.keras.optimizers as opt\n\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.models import Sequential, Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.models import model_from_json\n\nnp.random.RandomState(42)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = '../input/Kannada-MNIST/'\n\ntrain_path = data_path + 'train.csv'\ntest_path = data_path + 'test.csv'\nsample_path = data_path + 'sample_submission.csv'\n\nsave_path = ''\nload_path = '../input/kennada-mnist-pretrained-model/'\n\ntrain_df = pd.read_csv(train_path)\ntest_df = pd.read_csv(test_path)\nsample_df = pd.read_csv(sample_path)\n\n# convert dataframes to numpy matricies\nX = train_df.drop('label', axis=1).to_numpy()\ny = train_df['label'].to_numpy()\nX_test = test_df.drop('id', axis=1).to_numpy()\n\n# reshape X's for keras and encode y using one-hot-vector-encoding\nX = X.reshape(-1, 28, 28, 1)\ny = to_categorical(y)\nX_test = X_test.reshape(-1, 28, 28, 1)\n\n# normalize the data to range(0, 1)\nX = X / 255\nX_test = X_test / 255\n\nX_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.1, random_state=42) \n\n# split to train and validation sets\n\nprint('Train shape = {} {}'.format(X_train.shape, y_train.shape))\nprint('Valid shape = {} {}'.format(X_valid.shape, y_valid.shape))\nprint('Test shape = {}'.format(X_test.shape))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model builder\ndef get_model():\n    return Sequential([\n        l.Conv2D(64, (3,3), padding='same', input_shape=(28, 28, 1)),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n        l.Conv2D(64,  (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n        l.Conv2D(64,  (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n\n        l.MaxPooling2D(2, 2),\n        l.Dropout(0.25),\n\n        l.Conv2D(128, (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n        l.Conv2D(128, (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n        l.Conv2D(128, (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n\n        l.MaxPooling2D(2,2),\n        l.Dropout(0.25),    \n\n        l.Conv2D(256, (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        l.LeakyReLU(alpha=0.1),\n        l.Conv2D(256, (3,3), padding='same'),\n        l.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),##\n        l.LeakyReLU(alpha=0.1),\n\n        l.MaxPooling2D(2,2),\n        l.Dropout(0.25),\n\n        l.Flatten(),\n        l.Dense(256),\n        l.LeakyReLU(alpha=0.1),\n\n        l.BatchNormalization(),\n        l.Dense(10, activation='softmax')\n    ])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizers = {\n    'sgd':          opt.SGD(),\n    'sgd+momentum': opt.SGD(nesterov=True),\n    'rmsprop':      opt.RMSprop(),\n    'adam':         opt.Adam(),\n}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1024\nepochs = 5\n\ndatagen_train = ImageDataGenerator(\n    rotation_range = 10,\n    width_shift_range = 0.25,\n    height_shift_range = 0.25,\n    shear_range = 0.1,\n    zoom_range = 0.4,\n    horizontal_flip = False\n)\n\ndatagen_val = ImageDataGenerator()\n\nlearning_rate_reduction = ReduceLROnPlateau( \n    monitor='loss',\n    factor=0.25,\n    patience=2,\n    verbose=1,\n    mode=\"auto\",\n    min_delta=0.0001,\n    cooldown=0,\n    min_lr=0.00001\n)\n\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)\n\nhistory = {}\n\nfor name, optimizer in optimizers.items():\n    model = get_model()\n    model.compile(\n        loss='categorical_crossentropy',\n        optimizer=optimizer,\n        metrics=['accuracy']\n    )\n    h = model.fit_generator(\n        datagen_train.flow(X_train, y_train, batch_size=batch_size),\n        steps_per_epoch=len(X_train)//batch_size,\n        epochs=epochs,\n        validation_data=datagen_val.flow(X_valid, y_valid),\n        validation_steps=50,\n        callbacks=[learning_rate_reduction, es],\n        verbose=1\n    )\n\n    history[name] = h.history\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"styles=[':','-.','--','-']\nplt.figure(figsize=(20, 7))\n    \nfor n, h in enumerate(history.values()):\n    val_acc = h['val_accuracy']\n    plt.plot(h['val_accuracy'], linestyle=styles[n])\n\nplt.title('Model validation accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(optimizers.keys(), loc='upper left')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"styles=[':','-.','--','-']\nplt.figure(figsize=(20, 7))\n    \nfor n, h in enumerate(history.values()):\n    val_acc = h['accuracy']\n    plt.plot(h['accuracy'], linestyle=styles[n])\n\nplt.title('Model train accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(optimizers.keys(), loc='upper left')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"styles=[':','-.','--','-']\nplt.figure(figsize=(20, 7))\n    \nfor n, h in enumerate(history.values()):\n    val_acc = h['loss']\n    plt.plot(h['loss'], linestyle=styles[n])\n\nplt.title('Model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(optimizers.keys(), loc='upper left')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}