{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"### Installing TensorFlow 2.0"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#!pip install --upgrade tensorflow-gpu","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Importing the dependencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom __future__ import print_function, absolute_import, division, unicode_literals\n\n%matplotlib inline\n%config InlineBackend.figure_format = \"retina\"\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\n\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TensorFlow setup"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"TensorFlow version: {tf.__version__}\")\nprint(f\"Keras version: {tf.keras.__version__}\")\nprint(f\"TensorFlow is executing eagerly: {tf.executing_eagerly()}\")\nprint(\"GPU is\",\"available.\" if tf.test.is_gpu_available() else \"unavailable.\")\nprint(f\"Initializing radom seeds..{tf.random.set_seed(1)}\")\nprint(f\"Enabling TensorFlow Device Debugger..\")\ntf.config.set_soft_device_placement(True)\ntf.debugging.set_log_device_placement(True)\nprint(\"No of physical devices available: {}\".format(len(tf.config.experimental.list_physical_devices())))\n%reload_ext tensorboard.notebook\n\nimport warnings; warnings.simplefilter(\"ignore\")\nprint(\"Done.!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reading the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir('../input/Kannada-MNIST/')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/Kannada-MNIST/train.csv')\ntest_df = pd.read_csv('../input/Kannada-MNIST/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import display, clear_output\nwith pd.option_context(\"display.max_rows\",10,\"display.max_columns\",1000):\n    display(train_df.head(5))\n    clear_output()\n    display(test_df.head(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_labels = train_df.iloc[:,0].values\ntrain_features = train_df.iloc[:,1:].values.reshape(-1,28,28,1)\ntest_features = test_df.iloc[:,1:].values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_features.shape)\nprint(train_labels.shape)\nprint(test_features.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train and Validation"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(train_features, train_labels,test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define distribution strategy"},{"metadata":{"trusted":true},"cell_type":"code","source":"strategy = tf.distribute.MirroredStrategy(cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Number of devices in parallel: {strategy.num_replicas_in_sync}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup input pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"BUFFER_SIZE = 10000\nBATCH_SIZE_PER_REPLICA = 64\nBATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def scale(image, label):\n    image = tf.cast(image, tf.float32)\n    image /= 255\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train)) \\\n                .map(scale, num_parallel_calls=tf.data.experimental.AUTOTUNE) \\\n                .cache() \\\n                .shuffle(BUFFER_SIZE) \\\n                .batch(BATCH_SIZE) \\\n                .repeat() \\\n                .prefetch(tf.data.experimental.AUTOTUNE)\n\nval_dataset = tf.data.Dataset.from_tensor_slices((X_val, y_val)) \\\n                .map(scale, num_parallel_calls=tf.data.experimental.AUTOTUNE) \\\n                .batch(BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_dataset.element_spec)\nprint(val_dataset.element_spec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for images, labels in train_dataset.take(1):\n    pass\n\ntrain_features = images.numpy()\ntrain_labels = labels.numpy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Vizualize"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style(\"whitegrid\")\n\nplt.figure(figsize=(20,18))\nfor idx in range(36):\n    plt.subplot(6,6,idx+1)\n    plt.imshow(np.squeeze(train_features[idx]),cmap=plt.cm.binary,interpolation='nearest')\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(True)\n    plt.title(train_labels[idx])\n    plt.colorbar()\nplt.suptitle(\"Kannada MNIST Examples\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"activation = 'relu'\npadding = 'same'\ngamma_initializer = 'uniform'\ninput_shape=(28,28,1)\n\nwith strategy.scope():\n    input_layer = tf.keras.layers.Input(shape=input_shape)\n    hidden=tf.keras.layers.Conv2D(16, (3,3), strides=1, padding=padding, activation = activation, name=\"conv1\")(input_layer)\n    hidden=tf.keras.layers.BatchNormalization(axis =1, momentum=0.1, epsilon=1e-5, gamma_initializer=gamma_initializer, name=\"batch1\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.1)(hidden)\n\n    hidden=tf.keras.layers.Conv2D(32, (3,3), strides=1, padding=padding,activation = activation, name=\"conv2\")(hidden)\n    hidden=tf.keras.layers.BatchNormalization(axis =1,momentum=0.15, epsilon=1e-5, gamma_initializer=gamma_initializer, name=\"batch2\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.15)(hidden)\n    hidden=tf.keras.layers.MaxPool2D(pool_size=2, strides=2, padding=padding, name=\"max2\")(hidden)\n\n    hidden=tf.keras.layers.Conv2D(64, (3,3), strides=1, padding =padding, activation = activation,  name=\"conv3\")(hidden)\n    hidden=tf.keras.layers.BatchNormalization(axis =1,momentum=0.17, epsilon=1e-5, gamma_initializer=gamma_initializer, name=\"batch3\")(hidden)\n    hidden=tf.keras.layers.MaxPool2D(pool_size=2, strides=2, padding=\"same\", name=\"max3\")(hidden)\n\n    hidden=tf.keras.layers.Conv2D(128, (3,3), strides=1, padding=padding, activation = activation, name=\"conv4\")(hidden)\n    hidden=tf.keras.layers.BatchNormalization(axis =1,momentum=0.15, epsilon=1e-5, gamma_initializer=gamma_initializer, name=\"batch4\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.15)(hidden)\n    \n    hidden=tf.keras.layers.Conv2D(256, (3,3), strides=1, padding =padding, activation = activation,  name=\"conv5\")(hidden)\n    hidden=tf.keras.layers.BatchNormalization(axis =1,momentum=0.17, epsilon=1e-5, gamma_initializer=gamma_initializer, name=\"batch5\")(hidden)\n    hidden=tf.keras.layers.MaxPool2D(pool_size=2, strides=2, padding=\"same\", name=\"max4\")(hidden)\n\n    hidden=tf.keras.layers.Flatten()(hidden)\n    hidden=tf.keras.layers.Dense(1024,activation = activation, name=\"Dense1\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.05)(hidden)\n    hidden=tf.keras.layers.Dense(512,activation = activation, name=\"Dense2\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.05)(hidden)\n    hidden=tf.keras.layers.Dense(256, activation = activation, name=\"Dense3\")(hidden)\n    hidden=tf.keras.layers.Dropout(0.03)(hidden)\n    output = tf.keras.layers.Dense(10, activation = \"softmax\")(hidden)\n\n    model = tf.keras.Model(inputs=input_layer, outputs=output)\n\n    model.compile(loss='sparse_categorical_crossentropy',\n                optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n                metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define the callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import clear_output\nclass PlotLearning(tf.keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.i = 0\n        self.x = []\n        self.loss = []\n        self.val_loss = []\n        self.acc = []\n        self.val_acc=[]\n        self.fig = plt.figure()\n        self.logs = []\n\n    def on_epoch_end(self, epoch, logs={}):      \n        self.logs.append(logs)\n        self.x.append(self.i)\n        self.loss.append(logs.get('loss'))\n        self.val_loss.append(logs.get('val_loss'))\n        self.acc.append(logs.get('accuracy'))\n        self.val_acc.append(logs.get('val_accuracy'))\n\n        self.i += 1\n        f, ax = plt.subplots(1, 2, figsize=(12,4), sharex=True)\n        ax = ax.flatten()\n        clear_output(wait=True)\n\n        ax[0].plot(self.x, self.loss, label=\"loss\", lw=2)\n        ax[0].plot(self.x, self.val_loss, label=\"val loss\")\n        ax[0].legend()\n        ax[0].grid(True)\n\n        ax[1].plot(self.x, self.acc, label=\"accuracy\", lw=2)\n        ax[1].plot(self.x, self.val_acc, label=\"val accuracy\")\n        ax[1].legend()\n        ax[1].grid(True)\n\n    plt.show();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint_dir = './training_checkpoints'\ncheckpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def decay(epoch):\n    if epoch < 5:\n        return 1e-3\n    elif epoch >= 5 and epoch < 10:\n        return 1e-4\n    else:\n        return 1e-5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.TensorBoard(log_dir='logs'),\n    tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n                                       save_weights_only=True,verbose=1),\n    tf.keras.callbacks.LearningRateScheduler(decay,verbose=1),\n    PlotLearning(),\n    tf.keras.callbacks.EarlyStopping(monitor='val_loss',verbose=1,patience=6)\n]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train the model "},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_dataset, \n                    validation_data=val_dataset, \n                    steps_per_epoch=512, \n                    epochs=15, \n                    callbacks=callbacks, \n                    verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls {checkpoint_dir}","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Evaluate the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))\neval_loss, eval_acc = model.evaluate(val_dataset,verbose=1,steps=10)\nprint('Eval loss: {:.2}, Eval Accuracy: {:.2}'.format(eval_loss, eval_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Vizualize in TensorBoard"},{"metadata":{"trusted":true},"cell_type":"code","source":"# %tensorboard --logdir=logs","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Export to SavedModel"},{"metadata":{"trusted":true},"cell_type":"code","source":"path = 'saved_model/'\nmodel.save(path, save_format='tf')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Make Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = tf.data.Dataset.from_tensors(tf.cast(test_features,tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.argmax(model.predict(test_data),axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submit to Competition"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('../input/Kannada-MNIST/sample_submission.csv')\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['label'] = predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8,6))\nsubmission[\"label\"].hist(bins=20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}