{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# TensorFlow and tf.keras\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator, Iterator\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow import keras\nfrom keras import regularizers\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nimport pywt\nfrom tqdm import tnrange, tqdm_notebook\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape is: \" + str(train.shape))\nprint(\"test shape is: \" + str(test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop(['label'], axis = 1)\nX_valid = test.drop(['id'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"original TRAIN shape: \" + str(X.shape))\nprint(\"original TEST shape: \" + str(X_valid.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def haar(block):\n    a = pywt.dwt2(block, 'db1')\n    return a","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = train['label'].values\n\nX_exp = []\nfor i in tnrange(train.shape[0]):\n    im = train.iloc[i][train.columns[1:]].values.reshape((28,28))\n    a,(b,c,d) = haar(im)\n    newim = np.zeros((14,14,4))\n    newim[:,:,0] = a\n    newim[:,:,1] = b\n    newim[:,:,2] = c\n    newim[:,:,3] = d\n    X_exp.append(newim)\nX = np.array(X_exp)\nX_exp = []\nfor i in tnrange(test.shape[0]):\n    im = test.iloc[i][test.columns[1:]].values.reshape((28,28))\n    a,(b,c,d) = haar(im)\n    newim = np.zeros((14,14,4))\n    newim[:,:,0] = a\n    newim[:,:,1] = b\n    newim[:,:,2] = c\n    newim[:,:,3] = d\n    X_exp.append(newim)\nX_valid = np.array(X_exp)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X[i][:,:,0], cmap=plt.cm.binary)\n    plt.xlabel(train.label[i])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(64, (3,3), padding='same', input_shape=(14, 14, 4)),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(64,  (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(64,  (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n\n    tf.keras.layers.MaxPooling2D(2, 2),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(128, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(128, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(128, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    \n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.25),    \n    \n    tf.keras.layers.Conv2D(256, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n    tf.keras.layers.Conv2D(256, (3,3), padding='same'),\n    tf.keras.layers.BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),##\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.25),\n    \n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(256),\n    tf.keras.layers.LeakyReLU(alpha=0.1),\n \n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(10, activation='softmax')\n])\n\nprint(model.summary())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"initial_learningrate=0.001#*0.3\nmodel.compile(optimizer=\n              #Adam(learning_rate=0.0003),\n              RMSprop(lr=initial_learningrate),\n             loss = 'sparse_categorical_crossentropy',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='accuracy', \n                                            patience=300, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reference source for ImageDataGenerator from other kernel:\n\nhttps://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n\nhttps://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_decay(epoch, initial_learningrate = 0.001):#lrv 0.0003\n    return initial_learningrate * 0.99 ** epoch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batchsize = 200\nepoch = 45\ntrain_datagen = ImageDataGenerator(#rescale=1./255.,\n                                   rotation_range=10,\n                                   width_shift_range=0.25,\n                                   height_shift_range=0.25,\n                                   shear_range=0.1,\n                                   zoom_range=0.25,\n                                   horizontal_flip=False)\n\n\n\n\nvalid_datagen = ImageDataGenerator(#rescale=1./255.,\n                                    horizontal_flip=False,\n                                    rotation_range=15,\n                                   width_shift_range=0.25,\n                                   height_shift_range=0.25,\n                                   shear_range=0.15,\n                                   zoom_range=0.25,\n                                    )\n\n\n# add early stopping\ncallback = tf.keras.callbacks.EarlyStopping(monitor='accuracy', patience=5)\n\n# fit model with generated data\n\n\nhistory = model.fit_generator(train_datagen.flow(X_train, Y_train, batch_size = batchsize),\n                   steps_per_epoch = 100, \n                    epochs = epoch,\n                   callbacks=[callback,\n                            LearningRateScheduler(lr_decay),\n                            lr\n                             ],\n                   validation_data=valid_datagen.flow(X_dev, Y_dev),\n                    validation_steps=50,\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yhat = model.predict(X_valid).argmax(axis=1)\nsubmission['label']=pd.Series(yhat)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import img_to_array, load_img\nimport h5py\n\nfrom keras.models import load_model\n\nmodel.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}