{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"#Imports\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nprint (tf.__version__)\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.optimizers import *\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\nimport itertools\n%matplotlib inline\n#Constants\nBATCHSIZE=256\nOUTPUT_SHAPE = 10\n#Hyperparameters\nDROPOUT=0.1\nDROPOUT_FINAL=0.5\nROTATION=0.11\nWIDTH_SHIFT=0.23\nHEIGHT_SHIFT=0.19\nZOOM=0.39\nSHEAR=0.11\nINITIAL_LEARNING_RATE=0.001\nDECAY=0.99\nBASE_FILTERS=64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Preprocessing\ndef normalize(array):\n    return array / 256.0\n\ndef  generate_xy(df):\n    #train/val\n    if 'label' in df.columns:\n        labels = np.array(df[[\"label\"]],dtype=np.int32)\n        df.drop('label', axis=1, inplace=True)\n        ids = None\n    else:\n        labels = None\n        ids = df[\"id\"].values\n        df.drop('id', axis=1, inplace=True)\n    \n    #data\n    data = np.array(df,dtype=np.float32)\n    data = normalize(data)\n    data = np.reshape(data,(-1,28,28,1))\n    return data, labels, ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Testing\ndef validate_dataset(data, expected_shape, labels=None):\n    #Test shape\n    assert (data.shape == expected_shape)\n    #Print out some examples, labels and ids\n    if labels is not None:\n        print (\"Label: \" + str(labels[0][0])) \n    plt.imshow(data[0].reshape([28,28]), cmap='gray')\n\ndef fit_one_batch(data, model):\n    train_x, train_y = data.next()\n    history = model.fit(train_x, train_y,\n                   epochs=100,\n                   shuffle=True,\n                    verbose=0)\n    test_scores = model.evaluate(train_x, train_y, verbose=2, batch_size=BATCHSIZE)\n    test_acc = test_scores[1]\n    assert (test_acc > 0.95)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Create datasets\ntrain_df = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest_df = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nval_df = pd.read_csv(\"../input/Kannada-MNIST/Dig-MNIST.csv\")\ntrain_x, train_y, _ = generate_xy(train_df)\ntest_x, _, ids = generate_xy(test_df)\nval_x, val_y, _ = generate_xy(val_df)\ngenerator = ImageDataGenerator(\n     featurewise_center=False,\n     featurewise_std_normalization=False,\n     rotation_range=ROTATION,\n     width_shift_range=WIDTH_SHIFT,\n     height_shift_range=HEIGHT_SHIFT,\n     shear_range=SHEAR,\n     zoom_range=ZOOM)\ntrain_gen = generator.flow(train_x, train_y, batch_size=BATCHSIZE)\nval_gen = generator.flow(val_x, batch_size=BATCHSIZE, shuffle=False)\ntest_gen = generator.flow(test_x, batch_size=BATCHSIZE, shuffle=False)\n\n#Validate\nbatch_x, batch_y = train_gen.next()\nvalidate_dataset(batch_x, (BATCHSIZE,28,28,1), labels=batch_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model\ndef layer(inputs, features=16, stride=1, dropout=0.1, map=(3,3)):\n    out = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(dropout)(out) \n    return out\n\ndef resnet_layer(inputs, features=16, stride=1, dropout=0.1, map=(3,3)):\n    out = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(dropout)(out) \n    \n    skip = layers.Conv2D(features, map, strides=stride, padding=\"same\")(inputs)\n    out = layers.Add()([skip, out])\n    return out\n\ndef lr_decay(epoch, lr):\n    return INITIAL_LEARNING_RATE * 0.99 ** epoch\n\ndef make_model(input_shape, output_shape, dropout=0.2):\n    inputs = keras.Input(shape=input_shape)\n    filters = BASE_FILTERS\n    out = resnet_layer(inputs, features=filters, stride=1, dropout=dropout)\n    out = resnet_layer(out, features=filters, stride=2, dropout=dropout)\n    filters = filters * 2\n    out = resnet_layer(out, features=filters, stride=1, dropout=dropout)\n    out = resnet_layer(out, features=filters, stride=2, dropout=dropout)\n    filters = filters * 2\n    out = resnet_layer(out, features=filters, stride=1, dropout=dropout)\n    out = resnet_layer(out, features=filters, stride=2, dropout=dropout)\n    filters = filters * 2\n    out = resnet_layer(out, features=filters, stride=1, dropout=dropout)\n    \n    out = layers.GlobalMaxPool2D()(out)\n    out = layers.Activation('relu')(out)\n    out = layers.Dropout(DROPOUT_FINAL)(out)\n    out = layers.Dense(output_shape, activation='softmax')(out)\n    \n    model = keras.Model(inputs=inputs, outputs=out)\n    opt = Adam(learning_rate=INITIAL_LEARNING_RATE, beta_1=0.9, beta_2=0.999, amsgrad=False)\n    model.compile(optimizer=opt, loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])\n    return model\n\ncallbacks = [ModelCheckpoint('model.pth', monitor='val_acc', save_best_only=True),\n             EarlyStopping(monitor='val_acc', patience=10, verbose=2, restore_best_weights=True),\n             #ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.5),\n             LearningRateScheduler(lr_decay)\n            ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create and test model\nimg_shape = train_x.shape[1:]\nmodel = make_model(img_shape, OUTPUT_SHAPE, dropout=DROPOUT)\n\n#Validate model\nfit_one_batch(train_gen, model) #Fails if it cannot fit on one batch\n\n#Print model info\nprint(model.count_params())\nkeras.utils.plot_model(model, 'my_first_model_with_shape_info.png', show_shapes=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tta(generator, x, y, model, num_passes=5, test=False):\n    dfs = []\n    \n    predictions = model.predict(x, batch_size=BATCHSIZE)\n    pred_core = pd.DataFrame(predictions)\n    dfs.append(pred_core)\n    \n    for i in range(num_passes):\n        predictions = model.predict_generator(generator)\n        pred_new = pd.DataFrame(predictions)\n        dfs.append(pred_new)\n    \n    result = pd.concat(dfs)\n    result = result.groupby(result.index).mean()\n    if not test:\n        result['label_pred'] = result.idxmax('columns')\n        result['label'] = y\n        confusion_mtx = confusion_matrix(result['label'], result['label_pred']) \n        \n        result['score'] = np.where(result['label'] == result['label_pred'], 1, 0)\n        acc = result['score'].sum() / float(result['score'].size)\n        print ('Acc with TTA: {}'.format(acc))\n        return [acc], None, confusion_mtx\n\n    else:\n        return None, result, None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_average(num_models=2):\n    \n    all_scores = []\n    final_sub = pd.DataFrame()\n    for i in range(num_models):\n        model = make_model(img_shape, OUTPUT_SHAPE, dropout=DROPOUT)\n        history = model.fit_generator(train_gen,\n                           epochs=50,\n                           validation_data=(val_x, val_y),\n                           shuffle=True,\n                           verbose=False,\n                           callbacks=callbacks)\n        train_scores = model.evaluate_generator(train_gen, verbose=2)\n        test_scores = model.evaluate(val_x, val_y, verbose=2, batch_size=BATCHSIZE)\n        tta_scores, _, confusion_mtx = tta(val_gen, val_x, val_y, model)\n        \n        \n        all_scores.append(test_scores+train_scores+tta_scores)\n        \n        \n        #Gather predictions\n        _, sub, _ = tta(test_gen, test_x, None, model, test=True)\n        #predictions = model.predict(test_x, batch_size=BATCHSIZE)\n        #sub = pd.DataFrame(predictions)\n        sub['id'] = ids\n        if final_sub.empty:\n            final_sub = sub\n        else:\n            fina_sub = pd.concat([final_sub, sub])\n    \n    plot_confusion_matrix(confusion_mtx, classes = range(10))\n    scores = pd.DataFrame(all_scores, columns=['loss_val', 'acc_val', 'loss_train', 'acc_train', 'acc_val_tta'])\n    return scores, sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Evaluate and generate submission.csv\nscores, sub = model_average(5)\nprint(scores.describe())\nsub = sub.groupby('id').mean()\nsub['label'] = sub.idxmax('columns')\nsub[['label']].to_csv('submission.csv', index=True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}