{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport os\nimport glob\nimport timeit\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If the version is not 2.4.0 then uncomment the following line of code and restart the kernel using `shift + ctrl + p`"},{"metadata":{"trusted":true},"cell_type":"code","source":"# !pip install tensorflow==2.4.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking if GPU is enabled"},{"metadata":{"trusted":true},"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Dataset"},{"metadata":{},"cell_type":"markdown","source":"Specifying the path for `train.csv`, `test.csv` and `submission.csv`"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\nsub = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visualising the train dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The data is present in CSV format with 784 pixel values for each 28x28 image ranging between 0 and 255. The first column of `train.csv` is `label` which will be dropped to obtain `train_y` and the remaining dataset would be `train_x`. "},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop(['label'], axis=1).values\ny = train['label'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Splitting the `train` dataset into `train` and `validation` to check model performance later."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, val_X, train_y, val_y = train_test_split(X, y, test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X_train shape: \", train_X.shape)\nprint(\"X_val shape: \", val_X.shape)\nprint(\"y_train shape: \", train_y.shape)\nprint(\"y_val shape: \", val_y.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Changing the shape of the dataset to fit CNN. The target labels need to be one-hot encoded."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = train_X.reshape(-1, 28, 28, 1).astype(\"float32\")/255.0\nval_X = val_X.reshape(-1, 28, 28, 1).astype(\"float32\")/255.0\ntest = test.values.reshape(-1, 28, 28, 1).astype(\"float32\")/255.0\ntrain_y = tf.keras.utils.to_categorical(train_y)\nval_y = tf.keras.utils.to_categorical(val_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"X_train shape: \", train_X.shape)\nprint(\"X_val shape: \", val_X.shape)\nprint(\"y_train shape: \", train_y.shape)\nprint(\"y_val shape: \", val_y.shape)\nprint(\"test shape: \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining Callback"},{"metadata":{},"cell_type":"markdown","source":"Using Cyclical Learning Rate (CLR) as one of the call backs to check the difference in model performance. "},{"metadata":{"trusted":true},"cell_type":"code","source":"! git clone https://github.com/bckenstler/CLR.git","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import *\nfrom CLR.clr_callback import *\n\n# using the triangular learning rate policy and\n#  base_lr (initial learning rate which is the lower boundary in the cycle) is 0.1\nclr_triangular = CyclicLR(mode='triangular')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class myCallback(tf.keras.callbacks.Callback):\n  def on_epoch_end(self, epoch, logs={}):\n    if(logs.get('accuracy')>0.9999):\n      print(\"\\nReached 99.99% accuracy so cancelling training!\")\n      self.model.stop_training = True\n        \ncallbacks = myCallback()\n\nlearning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss', patience = 3, verbose = 1, factor = 0.5, min_lr = 1e-6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building Model using CNN"},{"metadata":{},"cell_type":"markdown","source":"## Augmentation using tf.image"},{"metadata":{},"cell_type":"markdown","source":"-  if you have 1D integer encoded target, you can use sparse_categorical_crossentropy as loss function\n-  if you have one-hot encoded your target in order to have 2D shape (n_samples, n_class), you can use categorical_crossentropy"},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyperparameters\nIMG_SIZE = 28\nBATCH_SIZE = 64\nAUTOTUNE = tf.data.experimental.AUTOTUNE\nnum_classes = 10\n\n# applying random transformations to the images\ndef augment( image_label, seed):\n    image, label = image_label\n    image = tf.image.resize_with_crop_or_pad(\n        image,\n        IMG_SIZE + 6, \n        IMG_SIZE + 6\n    )\n    # Make a new seed\n    new_seed = tf.random.experimental.stateless_split(seed, num=1)[0,:]\n    # Random crop back to the original size\n    image = tf.image.stateless_random_crop(\n        image, \n        size=( BATCH_SIZE, IMG_SIZE, IMG_SIZE, 1), \n        seed=seed\n    )\n    # Random brightness\n    image = tf.image.stateless_random_brightness(\n        image, \n        max_delta=0.5, \n        seed=new_seed\n    )\n    image = tf.clip_by_value(\n        image, \n        0, \n        1)\n    \n    return image, label\n\ndef create_train_val(train_ds, val_ds):\n    train_ds = (\n    train_ds\n    .shuffle(1000)\n    .map(f, num_parallel_calls=AUTOTUNE)\n    .prefetch(AUTOTUNE)\n    )\n\n    val_ds = (\n      val_ds\n      .prefetch(AUTOTUNE)\n    )\n\n    return train_ds, val_ds\n\ndef create_model(train_ds, val_ds, epochs):\n    # training the model\n    model = tf.keras.Sequential([\n        layers.Conv2D(32, (3, 3), activation = 'relu', padding = 'same', input_shape=(28,28,1)),\n        layers.Conv2D(32, (3, 3), activation = 'relu', padding = 'same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(2, 2),\n        layers.Dropout(0.2),\n        layers.Conv2D(64, (3, 3), activation = 'relu', padding = 'same'),\n        layers.Conv2D(64, (3, 3), activation = 'relu', padding = 'same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(2, 2),\n        layers.Dropout(0.2),\n        layers.Conv2D(128, (3, 3), activation = 'relu', padding = 'same'),\n        layers.Conv2D(128, (3, 3), activation = 'relu', padding = 'same'),\n        layers.BatchNormalization(),\n        layers.MaxPooling2D(2, 2),\n        layers.Dropout(0.2),\n        layers.Flatten(),\n        layers.Dense(512, activation = 'relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.2),\n        layers.Dense(256, activation = 'relu'),\n        layers.BatchNormalization(),\n        layers.Dropout(0.35),\n    \n        layers.Dense(10, activation = 'softmax')\n    ])\n\n    model.compile(optimizer=Adam(0.1),\n                loss=tf.keras.losses.CategoricalCrossentropy(),\n                metrics=['accuracy'])\n    \n    print(model.summary())\n    \n    return model\n\n# Create a generator\nrng = tf.random.Generator.from_seed(123, alg='philox')\n\n# A wrapper function for updating seeds\ndef f(x, y):\n    seed = rng.make_seeds(2)[0]\n    image, label = augment((x, y), seed)\n    return image, label\n\n# converting nd-array to tensors\ntrain_ds = tf.data.Dataset.from_tensor_slices((train_X, train_y)).batch(BATCH_SIZE)\nval_ds = tf.data.Dataset.from_tensor_slices((val_X, val_y)).batch(BATCH_SIZE)\ntest_ds = tf.data.Dataset.from_tensor_slices((test)).batch(BATCH_SIZE)\n\ntrain_ds, val_ds = create_train_val(train_ds, val_ds)\n\n# modeling\ncnn_model = create_model(train_ds, val_ds, 100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Fitting the above model : CNN-1"},{"metadata":{"trusted":true},"cell_type":"code","source":"startTime = timeit.default_timer()\nhistory = cnn_model.fit(\n    train_ds,\n    steps_per_epoch = train_X.shape[0] // BATCH_SIZE,\n    epochs = 50,\n    validation_data = val_ds,\n    validation_steps = val_X.shape[0] // BATCH_SIZE,\n    callbacks = [callbacks, learning_rate_reduction]\n)\nelapsedTime = timeit.default_timer() - startTime\nprint(\"Time taken for the Network to train : \",elapsedTime)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time taken for the Network to train :  208.07509070000003\n\nNow checking how CLR works on the same network"},{"metadata":{"trusted":true},"cell_type":"code","source":"startTime = timeit.default_timer()\nhistory = cnn_model.fit(\n    train_ds,\n    steps_per_epoch = train_X.shape[0] // BATCH_SIZE,\n    epochs = 50,\n    validation_data = val_ds,\n    validation_steps = val_X.shape[0] // BATCH_SIZE,\n    callbacks = [clr_triangular]\n)\nelapsedTime = timeit.default_timer() - startTime\nprint(\"Time taken for the Network to train : \",elapsedTime)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time taken for the Network to train (with CLR) :  217.03970317499994\n\nThere is not much time difference in training the model with 2 different callbacks but the loss function has decresed more in the case of CLR."},{"metadata":{},"cell_type":"markdown","source":"# CNN Model Predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = cnn_model.predict(test).argmax(axis=1)\nsub['Label'] = prediction\nsub.to_csv(\"MNIST_sub_cnn1.csv\", index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}