{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importing all relevent libraries and classes\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom kaggle_datasets import KaggleDatasets\nimport pandas as pd\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras import datasets, layers, models\nfrom keras.layers import Input\nfrom tensorflow.keras.layers import Layer\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:07:55.062088Z","iopub.execute_input":"2022-02-15T15:07:55.062369Z","iopub.status.idle":"2022-02-15T15:07:55.068443Z","shell.execute_reply.started":"2022-02-15T15:07:55.062337Z","shell.execute_reply":"2022-02-15T15:07:55.067678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Why we use nvidia-smi command ?\n\nThe NVIDIA System Management Interface (nvidia-smi) is a command line utility, based on top of the NVIDIA Management Library (NVML), intended to aid in the management and monitoring of NVIDIA GPU devices. ","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:07:55.766402Z","iopub.execute_input":"2022-02-15T15:07:55.767060Z","iopub.status.idle":"2022-02-15T15:07:56.541131Z","shell.execute_reply.started":"2022-02-15T15:07:55.767021Z","shell.execute_reply":"2022-02-15T15:07:56.540150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Pre-processing steps**","metadata":{}},{"cell_type":"markdown","source":"### Specification of using variable in code\n\n1. tf.constant : \ntf. constant is useful for asserting that the value can be embedded that way. If the argument dtype is not specified, then the type is inferred from the type of value . # Constant 1-D Tensor from a python list. tf.","metadata":{}},{"cell_type":"code","source":"# Imprting data from kaggle\ntrain = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\n\nx_train = train.drop(['label'], axis = 1)\ny_train = train['label']\n\nx_test = test\n\n# Devide the label in the try_train = tf.conaining, testing and validation set\ny_train = tf.constant(y_train) # converting the label into tensors\n\n# Divide the input in training, testing and validation set\nx_train = tf.constant(np.array(x_train), dtype = tf.float64)\nx_test = tf.constant(np.array(x_test), dtype = tf.float32)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:07:56.603741Z","iopub.execute_input":"2022-02-15T15:07:56.606280Z","iopub.status.idle":"2022-02-15T15:08:01.688480Z","shell.execute_reply.started":"2022-02-15T15:07:56.606231Z","shell.execute_reply":"2022-02-15T15:08:01.687665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# shape of training and test data\n\nprint('Shape of training data : {}'.format(x_train.shape))\nprint('Shape of test data : {}'.format(x_test.shape))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:01.690084Z","iopub.execute_input":"2022-02-15T15:08:01.690337Z","iopub.status.idle":"2022-02-15T15:08:01.696973Z","shell.execute_reply.started":"2022-02-15T15:08:01.690304Z","shell.execute_reply":"2022-02-15T15:08:01.695383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Display the Digits in the specified range\nplt.figure(figsize = (10,10))\ncount = 1\nid_no = 0\n\nfor i in range(7):\n    for j in range(7):\n        plt.subplot(7, 7, count)\n        plt.imshow(tf.constant(x_train[id_no, :], shape = [28, 28]), cmap = 'gray')\n        id_no += 1\n        count = count + 1\n        ","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:01.698853Z","iopub.execute_input":"2022-02-15T15:08:01.699127Z","iopub.status.idle":"2022-02-15T15:08:05.835737Z","shell.execute_reply.started":"2022-02-15T15:08:01.699094Z","shell.execute_reply":"2022-02-15T15:08:05.835012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Scaling the input \nx_train_s = (x_train - 127.5)/127.5\nx_test_s = (x_test - 127.5)/ 127.5\n\nprint('Shape of the scaled training data : {}'.format(x_train_s.shape))\nprint('Shape of the scaled test data : {}'.format(x_test_s.shape))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:05.837906Z","iopub.execute_input":"2022-02-15T15:08:05.838774Z","iopub.status.idle":"2022-02-15T15:08:05.860091Z","shell.execute_reply.started":"2022-02-15T15:08:05.838727Z","shell.execute_reply":"2022-02-15T15:08:05.859171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# splitting the training and test data\nfrom sklearn.model_selection import train_test_split\nx_train, x_valid, y_train, y_valid = train_test_split(x_train_s.numpy(), y_train.numpy(), test_size = 0.2)\nprint('Shape of the training data : {}'.format(x_train.shape))\nprint('Shape of the valid data : {}'.format(x_valid.shape))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:05.861686Z","iopub.execute_input":"2022-02-15T15:08:05.861983Z","iopub.status.idle":"2022-02-15T15:08:06.245174Z","shell.execute_reply.started":"2022-02-15T15:08:05.861931Z","shell.execute_reply":"2022-02-15T15:08:06.243543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reshaping the imagery\nx_train = tf.reshape(x_train, shape = [-1, 28, 28, 1])\nx_valid = tf.reshape(x_valid, shape = [-1, 28, 28, 1])\nx_test = tf.reshape(x_test, shape = [-1, 28, 28, 1])\nprint('Shape of the training data : {}'.format(x_train.shape))\nprint('Shape of the valid data : {}'.format(x_valid.shape))\n\n# encoding of labels \ny_train = tf.keras.utils.to_categorical(y_train)\ny_valid = tf.keras.utils.to_categorical(y_valid)\n#print('Encoded labels :',y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:06.246756Z","iopub.execute_input":"2022-02-15T15:08:06.247044Z","iopub.status.idle":"2022-02-15T15:08:06.524952Z","shell.execute_reply.started":"2022-02-15T15:08:06.247004Z","shell.execute_reply":"2022-02-15T15:08:06.524107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **1. Creating the model using sequential CNN**","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Model architecture >>\n\n","metadata":{}},{"cell_type":"markdown","source":"**Defining the model architecture using sequential CNN structure .. Now the question arises here >>**\n\nQ. What is sequential CNN ?\n","metadata":{}},{"cell_type":"code","source":"keras_model = tf.keras.models.Sequential()\ninput = Input(shape=(28,28,1))\n## Conv 2d-1\nconv2d_1 = layers.Conv2D(32,(3,3),strides=(1,1),padding='valid',kernel_initializer='glorot_uniform',name='Conv2D_1')(input)\nleaky_relu_1 = layers.LeakyReLU(alpha=0.25)(conv2d_1)\nbatch_norm_1 = layers.BatchNormalization()(leaky_relu_1)\n## Conv 2d-2\nconv2d_2 = layers.Conv2D(48,(3,3), strides=(1,1),padding='valid',kernel_initializer='glorot_uniform',name='Conv2D_2')(batch_norm_1)\nleaky_relu_2 = layers.LeakyReLU(alpha=0.25)(conv2d_2)\nbatch_norm_2 = layers.BatchNormalization()(leaky_relu_2)\nmax_pool_1 = layers.MaxPool2D()(batch_norm_2)\n## Conv 2d-3\nconv2d_3 = layers.Conv2D(64,(3,3), strides=(1,1),padding='valid',kernel_initializer='glorot_uniform')(max_pool_1)\nleaky_relu_3 = layers.LeakyReLU(alpha=0.25)(conv2d_3)\nbatch_norm_3 = layers.BatchNormalization()(leaky_relu_3)\n## Conv 2d-4\nconv2d_4 = layers.Conv2D(96,(3,3),strides=(1,1),padding='same',kernel_initializer='glorot_uniform')(batch_norm_3)\nleaky_relu_4 = layers.LeakyReLU(alpha=0.25)(conv2d_4)\nbatch_norm_4 = layers.BatchNormalization()(leaky_relu_4)\nmax_pool_2 = layers.MaxPool2D()(batch_norm_4)\n## Conv 2d-5\nconv2d_5 = layers.Conv2D(128,(3,3),strides=(1,1),padding='valid',kernel_initializer='glorot_uniform')(max_pool_2)\nleaky_relu_5 = layers.LeakyReLU(alpha=0.25)(conv2d_5)\nbatch_norm_5 = layers.BatchNormalization()(leaky_relu_5)\n## Conv 2d-6\nconv2d_6 = layers.Conv2D(256,(3,3),strides=(1,1),padding='valid',kernel_initializer='glorot_uniform')(batch_norm_5)\nleaky_relu_6 = layers.LeakyReLU(alpha=0.25)(conv2d_6)\nbatch_norm_6 = layers.BatchNormalization()(leaky_relu_6)\n## flatten & Dense\nflatten_1 = layers.Flatten()(batch_norm_6)\ndense_1 = layers.Dense(64)(flatten_1)\nleaky_relu_7 = layers.LeakyReLU(alpha=0.25)(dense_1)\nbatch_norm_7 = layers.BatchNormalization()(leaky_relu_7)\n## Dense\ndense_2 = layers.Dense(10)(batch_norm_7)\nbatch_norm_8 = layers.BatchNormalization()(dense_2)\n## activation\nactivation = layers.Activation('softmax')(batch_norm_8)\nkeras_model = models.Model(inputs=input, outputs=activation)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:06.526640Z","iopub.execute_input":"2022-02-15T15:08:06.526957Z","iopub.status.idle":"2022-02-15T15:08:06.679316Z","shell.execute_reply.started":"2022-02-15T15:08:06.526905Z","shell.execute_reply":"2022-02-15T15:08:06.678578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## summary of the model\nkeras_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:08:06.680654Z","iopub.execute_input":"2022-02-15T15:08:06.680893Z","iopub.status.idle":"2022-02-15T15:08:06.700613Z","shell.execute_reply.started":"2022-02-15T15:08:06.680860Z","shell.execute_reply":"2022-02-15T15:08:06.699966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now as we move further the 2nd question arises....\n\nQ. What is model compilation step ? ⏭\n\nA. The compilation is the final step in creating a model. Once the compilation is done, we can move on to training phase.","metadata":{}},{"cell_type":"code","source":"## compile the model\nkeras_model.compile(\n                    optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001),\n                    loss = tf.keras.losses.categorical_crossentropy,\n                    metrics = ['accuracy']\n                    )","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:58.396726Z","iopub.execute_input":"2022-02-15T15:55:58.397199Z","iopub.status.idle":"2022-02-15T15:55:58.412653Z","shell.execute_reply.started":"2022-02-15T15:55:58.397160Z","shell.execute_reply":"2022-02-15T15:55:58.411985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"Q. What is model callbacks ? ⏭\n\nA. model callbacks can help you fix bugs more quickly, and can help you build better models. They can help you visualize how your model's training is going, and can even help prevent overfitting by implementing early stopping or customizing the learning rate on each iteration.","metadata":{}},{"cell_type":"code","source":"# Initialize callbacks\ncallback_1 = EarlyStopping(patience = 3, restore_best_weights = True, monitor = 'val_acc', verbose = 1)\ncallback_2 = tf.keras.callbacks.ModelCheckpoint(\n    filepath='weight.h5', monitor='val_loss', verbose=1, save_best_only=True,\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:55:59.816716Z","iopub.execute_input":"2022-02-15T15:55:59.817027Z","iopub.status.idle":"2022-02-15T15:55:59.822911Z","shell.execute_reply.started":"2022-02-15T15:55:59.816991Z","shell.execute_reply":"2022-02-15T15:55:59.821980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data augmentation steps","metadata":{}},{"cell_type":"code","source":"data_traingen = ImageDataGenerator(\n                    rotation_range = 45,\n                    width_shift_range = 7,\n                    height_shift_range = 7,\n                    zoom_range = 1.1\n                    )\n\ndata_validgen = ImageDataGenerator(\n                    rotation_range = 45,\n                    width_shift_range = 7,\n                    height_shift_range = 7,\n                    zoom_range = 1.1\n                    )\n\ntrain_data = data_traingen.flow(x_train, y_train, batch_size = 32, shuffle = True)\nvalid_data = data_validgen.flow(x_valid, y_valid, batch_size = 32, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:01.016559Z","iopub.execute_input":"2022-02-15T15:56:01.016818Z","iopub.status.idle":"2022-02-15T15:56:01.072725Z","shell.execute_reply.started":"2022-02-15T15:56:01.016788Z","shell.execute_reply":"2022-02-15T15:56:01.071997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## model training\nhistory = keras_model.fit(train_data, epochs=30, validation_data = valid_data, callbacks = [callback_1, callback_2])","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:56:06.532272Z","iopub.execute_input":"2022-02-15T15:56:06.533017Z","iopub.status.idle":"2022-02-15T16:05:59.001042Z","shell.execute_reply.started":"2022-02-15T15:56:06.532965Z","shell.execute_reply":"2022-02-15T16:05:58.999319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.__version__","metadata":{"execution":{"iopub.status.busy":"2022-02-15T15:53:46.957554Z","iopub.execute_input":"2022-02-15T15:53:46.958330Z","iopub.status.idle":"2022-02-15T15:53:46.963187Z","shell.execute_reply.started":"2022-02-15T15:53:46.958291Z","shell.execute_reply":"2022-02-15T15:53:46.962514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model evaluation :\n\nevaluate() is for evaluating the already trained model using the validation (or test) data and the corresponding labels. Returns the loss value and metrics values for the model.","metadata":{}},{"cell_type":"code","source":"print('Evaluation on training data :',keras_model.evaluate(x_train, y_train))\nprint('Evaluation on validation data :',keras_model.evaluate(x_valid, y_valid))","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:05:59.002965Z","iopub.execute_input":"2022-02-15T16:05:59.003293Z","iopub.status.idle":"2022-02-15T16:06:05.916156Z","shell.execute_reply.started":"2022-02-15T16:05:59.003253Z","shell.execute_reply":"2022-02-15T16:06:05.915374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## plotting the history\ntrain_acc = history.history['accuracy']\nval_acc = history.history['val_accuracy']\ntrain_loss = history.history['loss']\nval_loss = history.history['val_loss']\n\nimport matplotlib.pyplot as plt\n\nepochs = range(1, len(train_acc) + 1)\n\nplt.plot(epochs, train_acc, 'b', label = 'Train acc')\nplt.plot(epochs, val_acc, 'r', label = 'Valid acc')\nplt.title('Training and Validation accuracy')\nplt.legend()\n","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:05.917469Z","iopub.execute_input":"2022-02-15T16:06:05.918223Z","iopub.status.idle":"2022-02-15T16:06:06.128486Z","shell.execute_reply.started":"2022-02-15T16:06:05.918183Z","shell.execute_reply":"2022-02-15T16:06:06.127796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.plot(epochs, train_loss, 'b', label = 'Train loss')\nplt.plot(epochs, val_loss, 'r', label = 'Valid loss')\nplt.title('Training and Validation loss')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:06.130667Z","iopub.execute_input":"2022-02-15T16:06:06.130954Z","iopub.status.idle":"2022-02-15T16:06:06.374616Z","shell.execute_reply.started":"2022-02-15T16:06:06.130906Z","shell.execute_reply":"2022-02-15T16:06:06.373828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model prediction","metadata":{}},{"cell_type":"code","source":"## predction on some sample\n\nimg = np.array(x_test[0:5])\npred = keras_model.predict(img)","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:06.375748Z","iopub.execute_input":"2022-02-15T16:06:06.376305Z","iopub.status.idle":"2022-02-15T16:06:06.567570Z","shell.execute_reply.started":"2022-02-15T16:06:06.376261Z","shell.execute_reply":"2022-02-15T16:06:06.566786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Display the results \nplt.figure(figsize=(10,10))\n\ncount = 1\nid_no = 0\n\nfor i in range(5):\n    for j in range(5):\n        if id_no == 5:\n            break\n        else:\n            plt.subplot(5,5, count)\n            plt.title(pred[id_no])\n            plt.imshow(tf.constant(x_test[id_no, :], shape = [28, 28]), cmap = 'gray')\n            id_no += 1\n            count = count + 1","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:06.570692Z","iopub.execute_input":"2022-02-15T16:06:06.572088Z","iopub.status.idle":"2022-02-15T16:06:07.062828Z","shell.execute_reply.started":"2022-02-15T16:06:06.572046Z","shell.execute_reply":"2022-02-15T16:06:07.062151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title(pred[1])\nplt.imshow(tf.constant(x_test[1, :], shape = [28, 28]), cmap = 'gray')","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:07.063955Z","iopub.execute_input":"2022-02-15T16:06:07.064801Z","iopub.status.idle":"2022-02-15T16:06:07.247434Z","shell.execute_reply.started":"2022-02-15T16:06:07.064759Z","shell.execute_reply":"2022-02-15T16:06:07.246774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = keras_model.predict(x_test)\npred = np.argmax(pred, axis = 1)\nsub = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\nsub['Label'] = pred\nsub.to_csv('submission.csv', index = False)\nsub.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-15T16:06:35.917365Z","iopub.execute_input":"2022-02-15T16:06:35.917703Z","iopub.status.idle":"2022-02-15T16:06:37.979532Z","shell.execute_reply.started":"2022-02-15T16:06:35.917668Z","shell.execute_reply":"2022-02-15T16:06:37.978798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}