{"cells":[{"metadata":{},"cell_type":"markdown","source":"In this notebook I try the multi class classification of the Kannada MNIST data set using a TPU. It is my first experiment where I try to implement the pipelining of tensorflow for TPU in a meaningful way."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#additional imports\nimport PIL.Image, PIL.ImageFont, PIL.ImageDraw\nfrom matplotlib import pyplot as plt\nimport os, re, time, json\nfrom keras import layers,models\nfrom keras.layers.core import Dense\nfrom keras.layers import Conv2D, MaxPool2D, Flatten\nfrom keras.regularizers import l2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\nimport tensorflow as tf\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection. No parameters necessary if TPU_NAME environment variable is set. On Kaggle this is always the case.\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n# select the appropriate distribution strategy\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# important parameters\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync # the global batchsize\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load the data from csv to a pandas dataframe\ntrain_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\") \nprint(train_df)\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#prepare the target-values an call it 'labels'\ny_train_df = train_df['label']#seperate the label-colummn as the target-column y_train_df\ntraining_labels = y_train_df\nx_train_df = train_df.drop(labels = ['label'],axis = 1)\n# call the x_train_df \"images\" an scale it from [0..255] to [0..1]\ntraining_images = x_train_df/255\ntraining_images = training_images.values.astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create the train- and validation-data for training only\nfrom sklearn.model_selection import train_test_split\ntraining_images,val_images,training_labels,val_labels = train_test_split(training_images, training_labels, test_size=0.1, random_state=None )\nprint(training_images.shape)\nprint(training_labels.shape)\nprint(val_images.shape)\nprint(val_labels.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# like to see the first training_image\nmyfirstdigit = training_images[0]\n#print(myfirstdigit)\nplt.imshow(myfirstdigit.reshape(28,28),cmap=plt.cm.binary)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# like to see the last training image\nmyfirstdigit = training_images[4860]\n#print(myfirstdigit)\nplt.imshow(myfirstdigit.reshape(28,28),cmap=plt.cm.binary)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# the validation data \"Dig-Mnist\" are for the evaluation\nfinal_val_set_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/Dig-MNIST.csv\") \nprint(final_val_set_df)\n#prepare the validation-target-values and call it 'labels'\nfinal_y_val_df = final_val_set_df['label']#seperate the label-cloumn as the target-column y_train_df\nfinal_val_labels = final_y_val_df\nfinal_x_val_df = final_val_set_df.drop(labels = ['label'],axis = 1)\n# call the x_train_df \"images\" an scale it from [0..255] to [0..1]\nfinal_val_images = final_x_val_df/255\nfinal_val_images = final_val_images.values.astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv') # \"dtype=str\" is importend for the later flow_from_dataframe-method\nsubmission_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build tensorflow input pipelines. The tf.data API makes it possible to handle large amounts of data, read from different data formats.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"# build two input pipeline: one for the train images,labels and one for the validation images,labels\n\ndef get_training_dataset(the_images,the_labels,batch_size):\n    # convert the training-data (images,labels) to a tf.dataset\n    dataset = tf.data.Dataset.from_tensor_slices((the_images,the_labels))\n    # shuffle, repeat and batch the samples\n    dataset = dataset.cache()# a small dataset can be cached in RAM\n    dataset = dataset.shuffle(1000, reshuffle_each_iteration = True)\n    dataset = dataset.repeat()# this ist manadatory for keras at this point\n    dataset = dataset.batch(batch_size, drop_remainder=True)\n    dataset = dataset.prefetch(-1)# fetch new batches while training on the current one\n    # return the dataset\n    return dataset\n\ndef get_validation_dataset(the_images,the_labels,batch_size):\n    # convert the training-data (images,labels) to a tf.dataset\n    dataset = tf.data.Dataset.from_tensor_slices((the_images,the_labels))\n    # shuffle, repeat and batch the samples\n    dataset = dataset.cache()# a small dataset can be cached in RAM\n    dataset.shuffle(1000, reshuffle_each_iteration = True)\n    dataset = dataset.repeat()# this ist manadatory for keras at this point\n    dataset = dataset.batch(batch_size, drop_remainder=True)\n    dataset = dataset.prefetch(-1)# fetch new batches while training on the current one\n    # return the dataset\n    return dataset\n\n\ndef get_final_validation_dataset(the_images,the_labels,batch_size):\n    # convert the training-data (images,labels) to a tf.dataset\n    dataset = tf.data.Dataset.from_tensor_slices((the_images,the_labels))\n    # shuffle, repeat and batch the samples\n    dataset = dataset.cache()# a small dataset can be cached in RAM\n    dataset.shuffle(1000, reshuffle_each_iteration = True)\n    dataset = dataset.repeat()# this ist manadatory for keras at this point\n    dataset = dataset.batch(batch_size, drop_remainder=True)\n    dataset = dataset.prefetch(-1)# fetch new batches while training on the current one\n    # return the dataset\n    return dataset\n\n# After I had problems with the computing of the private score by submission I propagate\n# the test_images direktly without making a dataset from them.\n# Fortunately, it works.\n\n#def get_test_dataset(the_images,batch_size):# only images, no labels => for prediction\n    # convert the training-data (images,labels) to a tf.dataset\n    #dataset = tf.data.Dataset.from_tensor_slices((the_images))\n    # shuffle, repeat and batch the samples\n    #dataset = dataset.cache()# a small dataset can be cached in RAM\n    #dataset = dataset.repeat()# this ist manadatory for keras at this point\n    #dataset = dataset.batch(batch_size, drop_remainder=True)\n    #dataset = dataset.prefetch(-1)# fetch new batches while training on the current one\n    # return the dataset\n    #return dataset\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# instantiate the datasets now\ntraining_dataset = get_training_dataset(training_images,training_labels,BATCH_SIZE)\nval_dataset=get_validation_dataset(val_images,val_labels,BATCH_SIZE)\nfinal_validation_dataset = get_final_validation_dataset(final_val_images,final_val_labels,BATCH_SIZE)\n\n# test_dataset = get_test_dataset(test_images,len(test_images))#all 5000 images # no need for prediction\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Activation funktion \"Leaky ReLU\" is one attempt to fix the “dying ReLU” problem. \n# Instead of the function being zero when x < 0, a leaky ReLU will instead have a \n# small negative slope (of 0,1, 0.01, or so).\n# In this model I use a parametrical LeakyRelu with paramater alpha: f(alpha,x)=alpha*x for x<0, =x for x>=0\n\n\ndef make_my_model ():\n    model = tf.keras.Sequential(\n    [tf.keras.layers.Reshape(input_shape=(28*28,),target_shape=(28,28,1),name=\"image\"),\n    \n    tf.keras.layers.Conv2D(filters=64, kernel_size=5, padding='same'), # no bias necessary before batch norm\n    tf.keras.layers.BatchNormalization(), # no batch norm scaling necessary before \"relu\"\n    tf.keras.layers.ReLU(), \n     \n    tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='same'), \n    tf.keras.layers.BatchNormalization(), \n    tf.keras.layers.ReLU(), \n     \n    tf.keras.layers.MaxPool2D((2,2)), \n    tf.keras.layers.Dropout(0.2),\n     \n    tf.keras.layers.Conv2D(filters=128, kernel_size=5, padding='same'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.LeakyReLU(alpha=0.1), # in the inner layer, my results are better with LeakyRelu\n    \n    tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding='same'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.LeakyReLU(alpha=0.1),  \n     \n    tf.keras.layers.MaxPool2D((2,2)),\n    tf.keras.layers.Dropout(0.2), \n    \n    tf.keras.layers.Flatten(),\n     \n    tf.keras.layers.Dense(256),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.LeakyReLU(alpha=0.1), \n        \n    tf.keras.layers.Dense(128),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.LeakyReLU(alpha=0.1), \n     \n    tf.keras.layers.Dropout(0.2),\n     \n    tf.keras.layers.Dense(32),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.LeakyReLU(alpha=0.1), \n    \n    tf.keras.layers.Dense(10, activation='softmax')# the last layer for the classes 0..9\n       \n    ])\n    \n    model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = make_my_model()\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training parameters\nEPOCHS = 100\nSTEPS_PER_EPOCH = 5400//BATCH_SIZE # 60,000 items in this dataset, 54000 train/6000 val\nprint(\"Steps per epoch: \", STEPS_PER_EPOCH)\n\n# parameters\n#BATCH_SIZE = 64 * strategy.num_replicas_in_sync # the global batchsize\n\nLEARNING_RATE =0.01\nif (strategy.num_replicas_in_sync == 1):\n    LEARNING_RATE_EXP_DECAY = 0.6\nelse: LEARNING_RATE_EXP_DECAY = 0.7\n\nLEARNING_RATE_DECAY = tf.keras.callbacks.LearningRateScheduler(\n                      lambda epoch: LEARNING_RATE * LEARNING_RATE_EXP_DECAY ** epoch,\n                    verbose=0)\n\nhistory = model.fit(training_dataset,\n                    steps_per_epoch=STEPS_PER_EPOCH,\n                    epochs=EPOCHS,\n                    callbacks=[LEARNING_RATE_DECAY],\n                    validation_data=(val_dataset),\n                    validation_steps=STEPS_PER_EPOCH,\n                    verbose=2\n                   )\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluate the model by using the \"Dig-MNIST.csv\"-data set for separate evaluation\nfinal_stats = model.evaluate(final_validation_dataset, steps=1)\nprint(\"Accuracy of the validation-set: \", final_stats[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_dict = history.history\nhistory_dict.keys()# look witch keys exist\nloss_values = history_dict['loss']\nacc_values = history_dict['accuracy']\nepochs = range(1, len(loss_values)+1)\nplt.plot(epochs,loss_values,'r', label='training loss')\nplt.plot(epochs,acc_values,'g',label='accuracy')\nplt.title('loss and accuracy')\nplt.xlabel('epochs')\nplt.ylabel('loss vs. accuracy')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load the data from csv to a pandas df\ntest_df = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")\nprint(test_df) # the test-data-set contains an \"id\"-column\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_df = test_df.drop(labels = ['id'],axis = 1)# drop the id-column for the test-dataset\nprint(x_test_df)\ntest_images = x_test_df/255\ntest_images = test_images.values.astype(np.float32)\ntest_images.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model.predict_classes(test_images)# I propagate the test_images directly to avoid erros by compute the private score\nsubmission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = results # entry the predicted result into the 'label' column\nsubmission\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}