{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#Imports\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nprint (tf.__version__)\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.optimizers import *\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nimport itertools\n%matplotlib inline\n#Constants\nBATCHSIZE=256\nOUTPUT_SHAPE = 10\n#Hyperparameters\nDROPOUT=0.15\nDROPOUT_FINAL=0.5\nROTATION=0.1\nWIDTH_SHIFT=0.2\nHEIGHT_SHIFT=0.15\nZOOM=0.3\nSHEAR=0.1\nINITIAL_LEARNING_RATE=0.001\nDECAY=0.99\nBASE_FILTERS=64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Preprocessing\ndef normalize(array):\n    return array / 256.0\n\ndef  generate_xy(df):\n    #train/val\n    if 'label' in df.columns:\n        labels = np.array(df[[\"label\"]],dtype=np.int32)\n        df.drop('label', axis=1, inplace=True)\n        ids = None\n    else:\n        labels = None\n        ids = df[\"id\"].values\n        df.drop('id', axis=1, inplace=True)\n    \n    #data\n    data = np.array(df,dtype=np.float32)\n    data = normalize(data)\n    data = np.reshape(data,(-1,28,28,1))\n    return data, labels, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Testing\ndef validate_dataset(data, expected_shape, labels=None, show_images=(3,3)):\n    #Test shape\n    assert (data.shape == expected_shape)\n    \n    #Print out some images\n    fig=plt.figure(figsize=(8, 8))\n    rows = show_images[0]\n    cols = show_images[1]\n    for i in range(rows*cols):\n        fig.add_subplot(rows, cols, i+1)\n        plt.imshow(data[i].reshape([28,28]), cmap='gray')\n\ndef fit_one_batch(data, model):\n    train_x, train_y = data.next()\n    history = model.fit(train_x, train_y,\n                   epochs=100,\n                   shuffle=True,\n                    verbose=0)\n    test_scores = model.evaluate(train_x, train_y, verbose=2, batch_size=BATCHSIZE)\n    test_acc = test_scores[1]\n    #Any model should be able to overfit to 95% accuracy in 100 epochs\n    assert (test_acc > 0.95)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Create datasets\ntrain_df = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest_df = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nval_df = pd.read_csv(\"../input/Kannada-MNIST/Dig-MNIST.csv\")\ntrain_x, train_y, _ = generate_xy(train_df)\ntest_x, _, ids = generate_xy(test_df)\nval_x, val_y, _ = generate_xy(val_df)\ngenerator = ImageDataGenerator(\n     featurewise_center=False,\n     featurewise_std_normalization=False,\n     rotation_range=ROTATION,\n     width_shift_range=WIDTH_SHIFT,\n     height_shift_range=HEIGHT_SHIFT,\n     shear_range=SHEAR,\n     zoom_range=ZOOM)\ntrain_gen = generator.flow(train_x, train_y, batch_size=BATCHSIZE)\nval_gen = generator.flow(val_x, batch_size=BATCHSIZE, shuffle=False)\ntest_gen = generator.flow(test_x, batch_size=BATCHSIZE, shuffle=False)\n\n#Validate\nbatch_x, batch_y = train_gen.next()\nvalidate_dataset(batch_x, (BATCHSIZE,28,28,1), labels=batch_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model\ndef layer(inputs, features=16, stride=1, dropout=0.1, map=(3,3)):\n    out = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(dropout)(out) \n    return out\n\ndef resnet_layer(inputs, features=16, stride=1, dropout=0.1, map=(3,3)):\n    out = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(dropout)(out) \n    \n    skip = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.Add()([skip, out])\n    return out\n\ndef lr_decay(epoch, lr):\n    return INITIAL_LEARNING_RATE * 0.99 ** epoch\n\ndef make_model(input_shape, output_shape, dropout=0.2):\n    inputs = keras.Input(shape=input_shape)\n    filters = BASE_FILTERS\n    out = layer(inputs, features=filters, stride=1, dropout=dropout)\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    out = layers.MaxPooling2D()(out)\n    filters = filters * 2\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    out = layers.MaxPooling2D()(out)\n    filters = filters * 2\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    out = layers.MaxPooling2D()(out)\n    filters = filters * 2\n    out = layer(out, features=filters, stride=1, dropout=dropout)\n    \n    out = layers.GlobalMaxPool2D()(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(DROPOUT_FINAL)(out)\n    out = layers.Dense(output_shape, activation='softmax')(out)\n    \n    model = keras.Model(inputs=inputs, outputs=out)\n    opt = Adam(learning_rate=INITIAL_LEARNING_RATE, beta_1=0.9, beta_2=0.999, amsgrad=False)\n    model.compile(optimizer=opt, loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])\n    return model\n\ncallbacks = [EarlyStopping(monitor='val_acc', patience=10, verbose=2, restore_best_weights=True),\n             LearningRateScheduler(lr_decay)\n            ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create and test model\nimg_shape = train_x.shape[1:]\nmodel = make_model(img_shape, OUTPUT_SHAPE, dropout=DROPOUT)\n\n#Validate model\nfit_one_batch(train_gen, model) #Fails if it cannot fit on one batch\n\n#Print model info\nprint(model.count_params())\nkeras.utils.plot_model(model, 'my_first_model_with_shape_info.png', show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes):\n    #plot a confusion matrix\n    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)\n    plt.title('Confusion matrix')\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test time augmentation\n# Predict on original images and on num_passes of generated images\n# Average prediction scores\ndef tta(generator, x, y, model, num_passes=5, test=False):\n    dfs = []\n    \n    predictions = model.predict(x, batch_size=BATCHSIZE)\n    pred_core = pd.DataFrame(predictions)\n    dfs.append(pred_core)\n    \n    for i in range(num_passes):\n        predictions = model.predict_generator(generator)\n        pred_new = pd.DataFrame(predictions)\n        dfs.append(pred_new)\n    \n    result = pd.concat(dfs)\n    result = result.groupby(result.index).mean()\n    if not test:\n        result['label_pred'] = result.idxmax('columns')\n        result['label'] = y\n        #Generate a confusion matrix\n        confusion_mtx = confusion_matrix(result['label'], result['label_pred']) \n        \n        result['score'] = np.where(result['label'] == result['label_pred'], 1, 0)\n        acc = result['score'].sum() / float(result['score'].size)\n        print ('Acc with TTA: {}'.format(acc))\n\n    else:\n        acc = None\n    \n    return [acc], result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train model\nmodel = make_model(img_shape, OUTPUT_SHAPE, dropout=DROPOUT)\nhistory = model.fit_generator(train_gen,\n                           epochs=50,\n                           validation_data=(val_x, val_y),\n                           shuffle=True,\n                           verbose=True,\n                           callbacks=callbacks,\n                           #class_weight=class_weights()\n                             )\n\n# Score model\ntrain_scores = model.evaluate_generator(train_gen, verbose=2)\ntest_scores = model.evaluate(val_x, val_y, verbose=2, batch_size=BATCHSIZE)\ntta_scores, result = tta(val_gen, val_x, val_y, model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Confusion matrix\nplot_confusion_matrix(confusion_matrix(result['label'], result['label_pred']), classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation accuracy values\nplt.plot(history.history['acc'])\nplt.plot(history.history['val_acc'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Gather predictions and generate submission.csv\n_, sub = tta(test_gen, test_x, None, model, test=True)\n#predictions = model.predict(test_x, batch_size=BATCHSIZE)\nsub['id'] = ids\nsub['label'] = sub.idxmax('columns')\nsub[['id', 'label']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}