{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# TensorFlow and tf.keras\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, Iterator\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow import keras\nfrom keras import regularizers\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tqdm import tnrange, tqdm_notebook\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape is: \" + str(train.shape))\nprint(\"test shape is: \" + str(test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop(['label'], axis = 1)\nX_valid = test.drop(['id'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"original TRAIN shape: \" + str(X.shape))\nprint(\"original TEST shape: \" + str(X_valid.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = train['label'].values\n\nX_exp = []\nfor i in tnrange(train.shape[0]):\n    im = train.iloc[i][train.columns[1:]].values.reshape((28,28))\n    newim = np.zeros((32,32))\n    newim[2:-2,2:-2] = im\n    X_exp.append(newim)\nX = np.array(X_exp)/255.\nX = X.reshape(X.shape[0],32,32,1)\n\nX_exp = []\nfor i in tnrange(test.shape[0]):\n    im = test.iloc[i][test.columns[1:]].values.reshape((28,28))\n    newim = np.zeros((32,32))\n    newim[2:-2,2:-2] = im\n    X_exp.append(newim)\nX_valid = np.array(X_exp)/255.\nX_valid = X_valid.reshape(X_valid.shape[0],32,32,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X[i][:,:,0], cmap=plt.cm.binary)\n    plt.xlabel(train.label[i])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CNN architechture\nf = 2**2\n\nmodel = tf.keras.Sequential([\n    # layer 1\n    tf.keras.layers.Conv2D(f*16,kernel_size=(3,3),padding=\"same\",activation='relu',\n                           kernel_initializer='he_uniform', \n                           input_shape=(32,32,1)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*16, (3,3), padding='same', \n                           activation ='relu',\n                           kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*16, (5,5), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    #tf.keras.layers.Dropout(0.15),\n    \n    tf.keras.layers.Conv2D(f*32, (3,3), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*32, (3,3), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*32, (5,5), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.15),\n    \n    # layer 3\n    tf.keras.layers.Conv2D(f*64,kernel_size=(3,3),padding=\"same\",activation='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*64,kernel_size=(3,3),padding=\"same\",activation='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    #tf.keras.layers.Conv2D(f*64,kernel_size=(5,5),padding=\"same\",activation='relu'),\n    #tf.keras.layers.Conv2D(f*64,kernel_size=(5,5),padding=\"same\",activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.15),\n    \n    # layer 4\n    #tf.keras.layers.Conv2D(f*128,kernel_size=(3,3),padding=\"same\",activation='relu'),\n    #tf.keras.layers.Conv2D(f*128,kernel_size=(3,3),padding=\"same\",activation='relu'),\n    #tf.keras.layers.BatchNormalization(),\n    #tf.keras.layers.MaxPooling2D(2,2),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(512, activation='relu'), #512\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(10, activation='softmax')\n])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_learningrate=0.001#*0.3\nmodel.compile(optimizer=\n              #Adam(learning_rate=0.0003),\n              RMSprop(lr=initial_learningrate),\n             loss = 'sparse_categorical_crossentropy',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='accuracy', \n                                            patience=300, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reference source for ImageDataGenerator from other kernel:\n\nhttps://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n\nhttps://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_decay(epoch, initial_learningrate = 0.001):#lrv 0.0003\n    return initial_learningrate * 0.99 ** epoch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.fftpack import dct, idct\n\ndef dct2(block):\n    return dct(dct(block.T, norm='ortho').T, norm='ortho')\n\ndef idct2(block):\n    return idct(idct(block.T, norm='ortho').T, norm='ortho')\n\nclass MyIterator(Iterator):\n  \"\"\"This is a toy example of a wrapper around ImageDataGenerator\"\"\"\n\n  def __init__(self, x, y, batch_size, shuffle, seed, **kwargs):\n    super().__init__(x.shape[0], batch_size, shuffle, seed)\n\n    # Load any data you need here (CSV, HDF5, raw stuffs). The code\n    # below is just a pseudo-code for demonstration purpose.\n    self.input_images = x\n    self.ground_truth = y\n\n    # Here is our beloved image augmentator <3\n    self.generator = ImageDataGenerator(**kwargs)\n\n  def _get_batches_of_transformed_samples(self, index_array):\n    \"\"\"Gets a batch of transformed samples from array of indices\"\"\"\n\n    # Get a batch of image data\n    batch_x = self.input_images[index_array].copy()\n    batch_y = self.ground_truth[index_array].copy()\n\n    # Transform the inputs and correct the outputs accordingly\n    for i, (x, y) in enumerate(zip(batch_x, batch_y)):\n        transform_params = self.generator.get_random_transform(x.shape)\n        batch_x[i] = self.generator.apply_transform(x, transform_params)\n        batch_x[i] = dct2(batch_x[i].reshape((32,32))).reshape((32,32,1))/1500 \n        batch_y[i] = y\n        \n    return batch_x, batch_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batchsize = 200\nepoch = 45\ncallback = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=5)\ntrain_datagen = MyIterator(X,\n                           Y,\n                           horizontal_flip=False,\n                           rotation_range=15,\n                           width_shift_range=0.25,\n                           height_shift_range=0.25,\n                           shear_range=0.15,\n                           zoom_range=0.25,batch_size = batchsize,shuffle=True,seed=0)\nval_datagen = MyIterator(X_dev,Y_dev,batch_size = batchsize,shuffle=False,seed=0)\nhistory = model.fit(train_datagen,\n                   steps_per_epoch = 100, \n                    epochs = epoch,\n                   callbacks=[callback,\n                            LearningRateScheduler(lr_decay),\n                            lr\n                             ],\n                   validation_data=val_datagen,\n                   validation_steps=50,\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_datagen = MyIterator(X_valid,submission.label.values,batch_size = batchsize,shuffle=False,seed=0)\n\nyhat = model.predict_generator(val_datagen).argmax(axis=1)\nsubmission['label']=pd.Series(yhat)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import img_to_array, load_img\nimport h5py\n\nfrom keras.models import load_model\n\nmodel.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}