{"cells":[{"metadata":{},"cell_type":"markdown","source":"Borrowing the CNN architecture from this excellent notebook:\nhttps://www.kaggle.com/kenanajk/understanding-cnns-with-kannada-mnist"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from pathlib import Path\n\n%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nimport numpy as np\nimport pandas as pd\n\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Dropout, Dense, Flatten, BatchNormalization, MaxPooling2D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tensorflow.keras.applications.resnet50 import ResNet50\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_schedule = LearningRateScheduler(lambda epoch: 1e-6 * 2**(epoch/2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = Path.cwd()/'..'/'input'/'Kannada-MNIST'/'train.csv'\n\nall_data = pd.read_csv(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = all_data.label\nx = all_data.iloc[:,1:].values\nx = x.reshape(x.shape[0], 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(rescale=1./255.,\n                                   rotation_range=10,\n                                   width_shift_range=0.25,\n                                   height_shift_range=0.25,\n                                   shear_range=0.1,\n                                   zoom_range=0.25,\n                                   horizontal_flip=False)\n\ntrain_generator = train_datagen.flow(x, y, batch_size=1024)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history):\n    #-----------------------------------------------------------\n    # Retrieve a list of list results on training and test data\n    # sets for each training epoch\n    #-----------------------------------------------------------\n    acc      = history.history[     'accuracy' ]\n    loss     = history.history[    'loss' ]\n\n    epochs   = range(len(acc)) # Get number of epochs\n\n    #------------------------------------------------\n    # Plot training and validation accuracy per epoch\n    #------------------------------------------------\n    plt.plot  ( epochs,     acc , label='Training')\n    plt.xlabel ('Epoch')\n    plt.ylabel ('Accuracy')\n    plt.legend ()\n    plt.title ('Training accuracy')\n    plt.figure()\n\n    #------------------------------------------------\n    # Plot training and validation loss per epoch\n    #------------------------------------------------\n    plt.plot  ( epochs,     loss, label='Training')\n    plt.xlabel ('Epoch')\n    plt.ylabel ('Loss')\n    plt.legend ()\n    plt.title ('Training loss'   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import Callback\nfrom tensorflow.keras import backend as K\n\nclass LR_Updater(Callback):\n    '''This callback is utilized to log learning rates every iteration (batch cycle)\n    it is not meant to be directly used as a callback but extended by other callbacks\n    ie. LR_Cycle\n    '''\n    def __init__(self, epoch_iterations):\n        '''\n        iterations = training batches\n        epoch_iterations = number of batches in one full training cycle\n        '''\n        self.epoch_iterations = epoch_iterations\n        self.trn_iterations = 0.\n        self.history = {}\n    def on_train_begin(self, logs={}):\n        self.trn_iterations = 0.\n        logs = logs or {}\n    def on_batch_end(self, batch, logs=None):\n        logs = logs or {}\n        self.trn_iterations += 1\n        K.set_value(self.model.optimizer.lr, self.setRate())\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n\nclass LR_Cycle(LR_Updater):\n    '''This callback is utilized to implement cyclical learning rates\n    it is based on this pytorch implementation https://github.com/fastai/fastai/blob/master/fastai\n    and adopted from this keras implementation https://github.com/bckenstler/CLR\n    '''\n    \n    def __init__(self, iterations, cycle_mult = 1):\n        '''\n        iterations = initial number of iterations in one annealing cycle\n        cycle_mult = used to increase the cycle length cycle_mult times after every cycle\n        for example: cycle_mult = 2 doubles the length of the cycle at the end of each cy$\n        '''\n        self.min_lr = 0\n        self.cycle_mult = cycle_mult\n        self.cycle_iterations = 0.\n        super().__init__(iterations)\n    \n    def setRate(self):\n        self.cycle_iterations += 1\n        if self.cycle_iterations == self.epoch_iterations:\n            self.cycle_iterations = 0.\n            self.epoch_iterations *= self.cycle_mult\n        decay_phase = np.pi*self.cycle_iterations/self.epoch_iterations\n        decay = (np.cos(decay_phase) + 1.) / 2.\n        return self.max_lr * decay\n    \n    def on_train_begin(self, logs={}):\n        super().on_train_begin(logs={})\n        self.cycle_iterations = 0.\n        self.max_lr = K.get_value(self.model.optimizer.lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(optimizer=Adam()):\n    model = Sequential()\n\n    model.add(Conv2D(64, kernel_size=3, activation='relu', input_shape=(28, 28, 1)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, kernel_size=3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.2))\n\n    model.add(Conv2D(128, kernel_size=3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(128, kernel_size=3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Conv2D(128, kernel_size=5, padding='same', activation='relu'))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.2))\n\n    model.add(Conv2D(256, kernel_size=3, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=(2, 2)))\n    model.add(Dropout(0.2))\n\n    model.add(Flatten())\n    model.add(Dense(256))\n    model.add(BatchNormalization())\n    model.add(Dense(128))\n    model.add(BatchNormalization())\n    model.add(Dense(10, activation='softmax'))\n    \n    model.compile(loss='sparse_categorical_crossentropy',\n              optimizer=optimizer,\n              metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(Adam(learning_rate=1e-3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cyclic_lr = LR_Cycle(59, cycle_mult=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit_generator(train_generator,\n                              epochs=31,\n                                callbacks=[cyclic_lr])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_path = Path.cwd()/'..'/'input'/'Kannada-MNIST'/'test.csv'\n\nall_data_test = pd.read_csv(test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = all_data_test.iloc[:,1:].values\nx_test = x_test.reshape(x_test.shape[0], 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(x_test/255.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.DataFrame({'id': all_data_test.id,\n                       'label': predictions})\noutput.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}