{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n# TensorFlow and tf.keras\nimport tensorflow as tf\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.losses import categorical_crossentropy\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow import keras\nfrom keras import regularizers\nfrom tensorflow.keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\ntest = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape is: \" + str(train.shape))\nprint(\"test shape is: \" + str(test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop(['label'], axis = 1)\nX_valid = test.drop(['id'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"original TRAIN shape: \" + str(X.shape))\nprint(\"original TEST shape: \" + str(X_valid.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reshape data before input to model\nX = X.values/255\nX_valid = X_valid.values/255\nY = train['label'].values\n\n# reshape\nX = X.reshape(X.shape[0],28,28,1)\nX_valid = X_valid.reshape(X_valid.shape[0],28,28,1)\n#Y = tf.keras.utils.to_categorical(Y)\n                                 \n\nprint(\"X data shape: \"+str(X.shape))\nprint(\"X_valid data shape: \"+str(X_valid.shape))\nprint(\"Y data shape: \"+str(Y.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_dev, Y_train, Y_dev = train_test_split(X, Y, test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X[i][:,:,0], cmap=plt.cm.binary)\n    plt.xlabel(train.label[i])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# CNN architechture\nf = 2**2\n\nmodel = tf.keras.Sequential([\n    # layer 1\n    tf.keras.layers.Conv2D(f*16,kernel_size=(3,3),padding=\"same\",activation='relu',\n                           kernel_initializer='he_uniform', \n                           input_shape=(28,28,1)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*16, (3,3), padding='same', \n                           activation ='relu',\n                           kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*16, (5,5), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    #tf.keras.layers.Dropout(0.15),\n    \n    tf.keras.layers.Conv2D(f*32, (3,3), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*32, (3,3), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*32, (5,5), padding='same', activation ='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.15),\n    \n    # layer 3\n    tf.keras.layers.Conv2D(f*64,kernel_size=(3,3),padding=\"same\",activation='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Conv2D(f*64,kernel_size=(3,3),padding=\"same\",activation='relu',\n                          kernel_regularizer=regularizers.l2(0.01)),\n    #tf.keras.layers.Conv2D(f*64,kernel_size=(5,5),padding=\"same\",activation='relu'),\n    #tf.keras.layers.Conv2D(f*64,kernel_size=(5,5),padding=\"same\",activation='relu'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(2,2),\n    tf.keras.layers.Dropout(0.15),\n    \n    # layer 4\n    #tf.keras.layers.Conv2D(f*128,kernel_size=(3,3),padding=\"same\",activation='relu'),\n    #tf.keras.layers.Conv2D(f*128,kernel_size=(3,3),padding=\"same\",activation='relu'),\n    #tf.keras.layers.BatchNormalization(),\n    #tf.keras.layers.MaxPooling2D(2,2),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(512, activation='relu'), #512\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(10, activation='softmax')\n])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model compiling\ninitial_learningrate=0.001#*0.3\nmodel.compile(optimizer=\n              #Adam(learning_rate=0.0003),\n              RMSprop(lr=initial_learningrate),\n             loss = 'sparse_categorical_crossentropy',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set a learning rate annealer\nlr = tf.keras.callbacks.ReduceLROnPlateau(monitor='acc', \n                                            patience=300, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Reference source for ImageDataGenerator from other kernel:\n\nhttps://www.kaggle.com/yassineghouzam/introduction-to-cnn-keras-0-997-top-6\n\nhttps://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lr_decay(epoch, initial_learningrate = 0.001):#lrv 0.0003\n    return initial_learningrate * 0.99 ** epoch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(#rescale=1./255.,\n                                   rotation_range=10,\n                                   width_shift_range=0.25,\n                                   height_shift_range=0.25,\n                                   shear_range=0.1,\n                                   zoom_range=0.25,\n                                   horizontal_flip=False)\n\n\n#datagen.fit(X_train)\n\nvalid_datagen = ImageDataGenerator(#rescale=1./255.,\n                                    horizontal_flip=False,\n                                    rotation_range=15,\n                                   width_shift_range=0.25,\n                                   height_shift_range=0.25,\n                                   shear_range=0.15,\n                                   zoom_range=0.25,\n                                    )\n\n# add early stopping\ncallback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)\n\n# fit model with generated data\nbatchsize = 512*4\nepoch = 45\n\nhistory = model.fit_generator(train_datagen.flow(X, Y, batch_size = batchsize),\n                   steps_per_epoch = 100, \n                    epochs = epoch,\n                   callbacks=[callback,\n                            LearningRateScheduler(lr_decay),\n                            lr\n                             ],\n                   validation_data=valid_datagen.flow(X_dev, Y_dev),\n                    validation_steps=50,\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train model\n#history = model.fit(X_train, Y_train, epochs=30,\n#          callbacks=[callback],\n#          validation_data=[X_dev,Y_dev]\n#         )\n\n# evaluate model performance\ntest_loss, test_acc = model.evaluate(X_dev, Y_dev,verbose=2)\n\nprint('\\nTest accuracy: ', test_acc)\nprint('\\nTest loss: ', test_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(1,2,1)\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train','Validation'])\n\nplt.subplot(1,2,2)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train','Validation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yhat = model.predict_classes(X_valid)\nsubmission['label']=pd.Series(yhat)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import img_to_array, load_img\nimport h5py\n\nfrom keras.models import load_model\n\nmodel.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}