{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport os\nimport numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt\nimport PIL.Image, PIL.ImageFont, PIL.ImageDraw\n#import tensorflow_addons as tfa\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nprint(\"Tensorflow version \" + tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 64\n\nTOTAL_DIGITS = 60000\nVALIDATION_DIGITS = 8000 # validation digits taken out of train.csv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### visualization utilities"},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"\"\"\"\nThis cell contains helper functions used for visualization\nand downloads only. You can skip reading it. There is very\nlittle useful Keras/Tensorflow code here.\n\"\"\"\n\n# Matplotlib config\nplt.rc('image', cmap='gray_r')\nplt.rc('grid', linewidth=0)\nplt.rc('xtick', top=False, bottom=False, labelsize='large')\nplt.rc('ytick', left=False, right=False, labelsize='large')\nplt.rc('axes', facecolor='F8F8F8', titlesize=\"large\", edgecolor='white')\nplt.rc('text', color='a8151a')\nplt.rc('figure', facecolor='F0F0F0')# Matplotlib fonts\n#MATPLOTLIB_FONT_DIR = os.path.join(os.path.dirname(plt.__file__), \"mpl-data/fonts/ttf\")\n\ndef dataset_to_numpy_util(dataset, N):\n    dataset = dataset.unbatch().batch(N)\n  \n    for digits, labels in dataset:\n        digits = digits.numpy()\n        labels = np.argmax(labels.numpy(), axis=1) # these were one-hot encoded in the dataset\n        break\n    \n    return digits, labels\n\n# create digits from local fonts for testing\ndef create_digits_from_local_fonts(n):\n    font_labels = []\n    img = PIL.Image.new('LA', (28*n, 28), color = (0,255)) # format 'LA': black in channel 0, alpha in channel 1\n    font1 = PIL.ImageFont.truetype(os.path.join(MATPLOTLIB_FONT_DIR, 'DejaVuSansMono.ttf'), 21)\n    font2 = PIL.ImageFont.truetype(os.path.join(MATPLOTLIB_FONT_DIR, 'STIXGeneral.ttf'), 21)\n    d = PIL.ImageDraw.Draw(img)\n    for i in range(n):\n        font_labels.append(i%10)\n        d.text((7+i*28,0 if i<10 else -4), chr(ord(u'೦')+i%10), fill=(255,255), font=font1 if i<10 else font2)\n        #d.text((7+i*28,0 if i<10 else -4), chr(ord(u'0')+i%10), fill=(255,255), font=font1 if i<10 else font2)\n    font_digits = np.array(img.getdata(), np.float32)[:,0] / 255.0 # black in channel 0, alpha in channel 1 (discarded)\n    font_digits = np.reshape(np.stack(np.split(np.reshape(font_digits, [28, 28*n]), n, axis=1), axis=0), [n, 28*28])\n    return font_digits, font_labels\n\n# utility to display a row of digits with their predictions\ndef display_digits(digits, predictions, labels, title, digits_per_row, row, n_rows):\n    if row==0: # set up the subplots on the first call\n        plt.subplots(figsize=(14,1.1*n_rows))\n        plt.tight_layout()\n    ax = plt.subplot(n_rows, 1, row+1) # index is 1-based\n    digits = np.reshape(digits, [digits_per_row, 28, 28])\n    digits = np.swapaxes(digits, 0, 1)\n    digits = np.reshape(digits, [28, 28*digits_per_row])\n    ax.set_yticks([])\n    ax.set_xticks([28*x+14 for x in range(digits_per_row)])\n    ax.set_xticklabels(predictions)\n    for i,t in enumerate(ax.get_xticklabels()):\n        if predictions[i] != labels[i]: t.set_color('red') # bad predictions in red\n    ax.imshow(digits)\n    ax.grid(None)\n    ax.set_title(title)\n\ndef display_rows_of_digits(digits, predictions, labels, title, n_rows, digits_per_row):\n    digits = np.reshape(digits[:n_rows*digits_per_row], [n_rows, digits_per_row, 28, 28])\n    predictions = np.reshape(predictions[:n_rows*digits_per_row], [n_rows, digits_per_row])\n    labels = np.reshape(labels[:n_rows*digits_per_row], [n_rows, digits_per_row])\n    for row in range(n_rows):\n        display_digits(digits[row], predictions[row], labels[row],\n                       title if row==0 else \"\", digits_per_row, row, n_rows)\n  \n# utility to display multiple rows of digits, sorted by unrecognized/recognized status\ndef display_top_unrecognized(digits, predictions, labels, n_rows, digits_per_row):\n    idx = np.argsort(predictions==labels) # sort order: unrecognized first\n    for row in range(n_rows):\n        display_digits(digits[idx][row*digits_per_row:(row+1)*digits_per_row],\n                       predictions[idx][row*digits_per_row:(row+1)*digits_per_row],\n                       labels[idx][row*digits_per_row:(row+1)*digits_per_row],\n                       \"{} sample validation digits out of {} with bad predictions in red and sorted first\".format(digits_per_row*n_rows,\n                                                                                                                   len(digits)) if row==0 else \"\",\n                       digits_per_row, row, n_rows)\n    \n# utility to display training and validation curves\ndef display_training_curves(training, validation, title, subplot):\n    if subplot%10==1: # set up the subplots on the first call\n        plt.subplots(figsize=(10,10), facecolor='#F0F0F0')\n        plt.tight_layout()\n    ax = plt.subplot(subplot)\n    ax.grid(linewidth=1, color='white')\n    ax.plot(training)\n    ax.plot(validation)\n    ax.set_title('model '+ title)\n    ax.set_ylabel(title)\n    ax.set_xlabel('epoch')\n    ax.legend(['train', 'valid.'])\n    \ndef display_lr_schedule(lr_fn, epochs):\n    X = range(epochs)\n    Y = np.array([lr_fn(x) for x in X])\n    min_lr = min(Y)\n    max_lr = max(Y)\n    title = \"Learning rate schedule\\n\"\n    title += \"max lr: {0:.4g}\".format(max_lr) + \"\\n\"\n    title += \"min lr: {0:.4g}\".format(min_lr)\n    plt.title(title)\n    plt.plot(X, Y)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_defaults = [tf.constant(0, dtype=tf.int32) for i in range(28*28+1)] # format: label, 28*28=784 pixel columns, all ints\ndef decode_csv_line(line, has_labels):    \n    data = tf.io.decode_csv(line, column_defaults)\n    first_col = data[0]\n    if has_labels:\n        label = tf.one_hot(first_col, 10)\n    else:\n        label = first_col # in fact just an index\n    pixels = data[1:]\n    image = tf.reshape(pixels, [28,28])\n    image = tf.cast(image, tf.float32) / 255.0\n    return image, label\n\n#def augment_rotate(image, label):\n#    # this one requires tensorflow_addons (not yet available on Kaggle)\n#    angles = tf.random.normal([tf.shape(image)[0]], stddev=0.4)\n#    image = tfa.image.rotate(tf.expand_dims(image, axis=-1), angles, interpolation='BILINEAR')\n#    return tf.squeeze(image, axis=-1), label\n\ndef augment(image, label):\n    std = 0.1\n    image = tf.expand_dims(image, axis=-1)\n    batch = tf.shape(image)[0]\n    dy1 = tf.random.normal([batch], stddev=std)\n    dx1 = tf.random.normal([batch], stddev=std)\n    dy2 = tf.random.normal([batch], stddev=std)\n    dx2 = tf.random.normal([batch], stddev=std)\n    y1 = tf.zeros([batch]) + dy1\n    x1 = tf.zeros([batch]) + dx1\n    y2 = tf.ones([batch]) + dy2\n    x2 = tf.ones([batch]) + dx2\n    box = tf.stack([y1, x1, y2, x2], axis=-1)\n    image = tf.image.crop_and_resize(image, box, tf.range(batch), [28, 28])\n    return tf.squeeze(image, axis=-1), label\n\ndef blur(image, label):\n    image = tf.expand_dims(image, axis=-1)\n    image = tf.image.resize(image, [56,56], method=tf.image.ResizeMethod.BICUBIC, antialias=True)\n    image = tf.image.resize(image, [28,28], method=tf.image.ResizeMethod.BICUBIC, antialias=True)\n    image = tf.squeeze(image, axis=-1)\n    return (image, label)\n\ndef load_dataset(filename, has_labels):\n    dataset = tf.data.TextLineDataset(filename)\n    dataset = dataset.skip(1) # header line\n    dataset = dataset.map(lambda line: decode_csv_line(line, has_labels), num_parallel_calls=AUTOTUNE)\n    return dataset \n\ndef get_training_dataset(filename, batch_size):\n    dataset = load_dataset(filename, has_labels=True).skip(VALIDATION_DIGITS)\n    dataset = dataset.cache()  # this small dataset can be entirely cached in RAM\n    dataset = dataset.shuffle(5000, reshuffle_each_iteration=True)\n    dataset = dataset.repeat()\n    dataset = dataset.batch(batch_size)\n    \n    dataset = dataset.map(blur)\n    dataset = dataset.map(augment)\n    \n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_validation_dataset(filename, batch_size):\n    dataset = load_dataset(filename, has_labels=True).take(VALIDATION_DIGITS)\n    dataset = dataset.cache()\n    dataset = dataset.batch(batch_size)\n    \n    dataset = dataset.map(blur)\n    \n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset\n\ndef get_submission_dataset(filename, batch_size):\n    dataset = load_dataset(filename, has_labels=False)\n    dataset = dataset.cache()\n    dataset = dataset.batch(batch_size)\n    \n    dataset = dataset.map(blur)\n    \n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = get_training_dataset(\"/kaggle/input/Kannada-MNIST/train.csv\", batch_size=BATCH_SIZE)\nvalid_dataset = get_validation_dataset(\"/kaggle/input/Kannada-MNIST/train.csv\", batch_size=VALIDATION_DIGITS) # one batch with everything\nsubmit_dataset = get_submission_dataset(\"/kaggle/input/Kannada-MNIST/test.csv\", batch_size=1000)\nsteps_per_epoch = (TOTAL_DIGITS - VALIDATION_DIGITS) // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"PER_ROW = 24\nN = PER_ROW*4\ntrain_digits, train_labels = dataset_to_numpy_util(train_dataset, N)\nvalid_digits, valid_labels = dataset_to_numpy_util(valid_dataset, N)\ndisplay_rows_of_digits(train_digits, train_labels, train_labels, \"training digits and their labels\",   N//PER_ROW, PER_ROW)\ndisplay_rows_of_digits(valid_digits, valid_labels, valid_labels, \"validation digits and their labels\", N//PER_ROW, PER_ROW)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Keras model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_model():\n    model = tf.keras.Sequential(\n      [\n        tf.keras.layers.Reshape(input_shape=(28,28), target_shape=(28, 28, 1), name=\"image\"),\n        \n        tf.keras.layers.SeparableConv2D(filters=12, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),\n        #tf.keras.layers.Conv2D(filters=12, kernel_size=3, padding='same', use_bias=False), # no bias necessary before batch norm\n        tf.keras.layers.BatchNormalization(scale=False, center=True), # no batch norm scaling necessary before \"relu\"\n        tf.keras.layers.Activation('relu'), # activation after batch norm\n          \n        tf.keras.layers.SeparableConv2D(filters=16, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2),\n        #tf.keras.layers.Conv2D(filters=12, kernel_size=3, padding='same', use_bias=False), # no bias necessary before batch norm\n        tf.keras.layers.BatchNormalization(scale=False, center=True), # no batch norm scaling necessary before \"relu\"\n        tf.keras.layers.Activation('relu'), # activation after batch norm\n\n        tf.keras.layers.SeparableConv2D(filters=20, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2), \n        #tf.keras.layers.Conv2D(filters=24, kernel_size=6, padding='same', use_bias=False, strides=2, name=\"XXX\"),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n          \n        tf.keras.layers.SeparableConv2D(filters=24, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2), \n        #tf.keras.layers.Conv2D(filters=24, kernel_size=6, padding='same', use_bias=False, strides=2, name=\"XXX\"),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n          \n        tf.keras.layers.SeparableConv2D(filters=28, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2), \n        #tf.keras.layers.Conv2D(filters=24, kernel_size=6, padding='same', use_bias=False, strides=2, name=\"XXX\"),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n\n        tf.keras.layers.SeparableConv2D(filters=32, kernel_size=6, padding='same', use_bias=False, strides=2, depth_multiplier=2),\n        #tf.keras.layers.Conv2D(filters=32, kernel_size=6, padding='same', use_bias=False, strides=2),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n          \n        tf.keras.layers.SeparableConv2D(filters=36, kernel_size=3, padding='same', use_bias=False, depth_multiplier=2), \n        #tf.keras.layers.Conv2D(filters=24, kernel_size=6, padding='same', use_bias=False, strides=2, name=\"XXX\"),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n\n        #tf.keras.layers.GlobalAveragePooling2D(),\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(100, use_bias=False),\n        tf.keras.layers.BatchNormalization(scale=False, center=True),\n        tf.keras.layers.Activation('relu'),\n        tf.keras.layers.Dropout(0.4), # Dropout on dense layer only\n\n        tf.keras.layers.Dense(10, activation='softmax')\n      ])\n\n    model.compile(optimizer='adam', # learning rate will be set by LearningRateScheduler\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model\n    \n#with strategy.scope():\nmodel = make_model()\n\n# print model layers\nmodel.summary()\n\n# set up learning rate decay\ndef lr_fn(epoch):\n    return LEARNING_RATE * LEARNING_RATE_EXP_DECAY**epoch\nlr_decay = tf.keras.callbacks.LearningRateScheduler(lr_fn, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 60\nLEARNING_RATE = 0.01\nLEARNING_RATE_EXP_DECAY = 0.95\ndisplay_lr_schedule(lr_fn, EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_dataset, steps_per_epoch=steps_per_epoch, validation_data=valid_dataset, validation_steps=1, epochs=EPOCHS, callbacks=[lr_decay])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"Final validation accuracy:\", history.history[\"val_accuracy\"][-1])\ndisplay_training_curves(history.history['loss'], history.history['val_loss'], 'loss', 211)\ndisplay_training_curves(history.history['accuracy'], history.history['val_accuracy'], 'accuracy', 212)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visual validation"},{"metadata":{"trusted":false},"cell_type":"code","source":"# recognize validation digits\ndigits, labels = dataset_to_numpy_util(valid_dataset, VALIDATION_DIGITS)\nprobabilities = model.predict(digits, steps=1)\npredicted_labels = np.argmax(probabilities, axis=1)\ndisplay_top_unrecognized(digits, predicted_labels, labels, n_rows=12, digits_per_row=24)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## generate submission"},{"metadata":{"_kg_hide-output":true,"trusted":false},"cell_type":"code","source":"results = None\nfor images, indices in submit_dataset:\n    predictions = model.predict(images)\n    df = pd.DataFrame({'id':indices.numpy(), 'label':np.argmax(predictions, axis=-1)})\n    results = results.append(df) if results is not None else df\nresults.to_csv(\"submission.csv\", index=False)\n!ls -al \"submission.csv\"\n!wc -l \"submission.csv\"\n!head \"submission.csv\" -n 15\n!echo \"...\"\n!tail \"submission.csv\" -n 15","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.3"}},"nbformat":4,"nbformat_minor":1}