{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom tensorflow import keras\nfrom keras.layers import Flatten\nfrom keras.layers import Conv2D\nfrom keras.layers import MaxPool2D\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import BatchNormalization\nfrom keras.layers import Average\nfrom keras.layers import Activation\nfrom keras.layers import GlobalAveragePooling2D\n\nfrom tensorflow.keras import layers\nprint(\"TF version\", tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nKMNIST_WIDTH, KMNIST_HEIGHT = 28, 28\ndef load_data(do_reshape=False, resize_shape=None, channel_num=1):\n    DMNIST_pd = pd.read_csv(\"/kaggle/input/Kannada-MNIST/Dig-MNIST.csv\")\n    train_pd = pd.read_csv(\"/kaggle/input/Kannada-MNIST/train.csv\")\n    test_pd = pd.read_csv(\"/kaggle/input/Kannada-MNIST/test.csv\")\n    x_train_np, y_train_np = train_pd.iloc[:,1:].to_numpy(), train_pd.iloc[:,0].to_numpy()\n    x_DMNIST_np, y_DMNIST_np = DMNIST_pd.iloc[:,1:].to_numpy(), DMNIST_pd.iloc[:,0].to_numpy()\n    \n    x_data=np.concatenate((x_train_np,x_DMNIST_np))\n    y_data=np.concatenate((y_train_np,y_DMNIST_np))\n    x_test_full = test_pd.iloc[:,1:].to_numpy()\n    \n    x_train_full, x_valid_full, y_train_full, y_valid_full = train_test_split(x_data, y_data,test_size=0.2, random_state=42)\n    x_train_full = x_train_full.astype('float32')\n    x_valid_full = x_valid_full.astype('float32')\n    x_test_full = x_test_full.astype('float32')\n    \n    # resize image for different NN\n    if resize_shape:\n        x_train_full = tf_img_resize(x_train_full, resize_shape)\n        x_valid_full = tf_img_resize(x_valid_full, resize_shape)\n        x_test_full = tf_img_resize(x_test_full, resize_shape)\n    # reshape to 2D for CNN\n    if do_reshape:\n        (w, h) = (KMNIST_WIDTH, KMNIST_HEIGHT) if not resize_shape else (resize_shape[0], resize_shape[1])\n        x_train_full = np.reshape(x_train_full, (-1, w, h, 1)).astype('float32')\n        x_valid_full = np.reshape(x_valid_full, (-1, w, h, 1)).astype('float32')\n        x_test_full = np.reshape(x_test_full, (-1, w, h, 1)).astype('float32')\n    if channel_num > 1:\n        x_train_full = np.repeat(x_train_full, channel_num, -1)\n        x_valid_full = np.repeat(x_valid_full, channel_num, -1)\n        x_test_full = np.repeat(x_test_full, channel_num, -1)\n        \n    return (x_train_full, y_train_full), (x_valid_full, y_valid_full), x_test_full\n\nimport cv2\ndef cv_img_resize(imgs, resize_shape):\n    def my_cv_resize(a):\n        return cv2.resize(a.reshape(28,28), resize_shape, interpolation=cv2.INTER_CUBIC)\n        #return tf.image.resize(a.reshape(28,28), (92,92), method=\"bicubic\")\n    return np.apply_along_axis(my_cv_resize, 1, imgs)\n    #return np.apply_along_axis(tf.image.resize, 0, imgs, resize, method=\"bicubic\")\n\ndef tf_img_resize(imgs, resize_shape):\n    def my_cv_resize(a):\n        #return cv2.resize(a.reshape(28,28), resize_shape, interpolation=cv2.INTER_CUBIC)\n        return tf.image.resize(a.reshape(28,28,1), [96,96], method=\"bicubic\")\n    return np.apply_along_axis(my_cv_resize, 1, imgs)\n    #return np.apply_along_axis(tf.image.resize, 0, imgs, resize, method=\"bicubic\")\n\n\n\ndef show_imgs(x_train, y_train, rows=4):\n    classes = [\"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\"]\n    print(classes)\n    for y, cls in enumerate(classes):\n        idxs = np.nonzero([i == y for i in y_train])\n        idxs = np.random.choice(idxs[0], rows)\n        for i , idx in enumerate(idxs):\n            plt_idx = i * len(classes) + y + 1\n            plt.subplot(rows, len(classes), plt_idx)\n            plt.imshow(x_train[idx].reshape((x_train.shape[1], x_train.shape[2],x_train.shape[3])))\n            plt.axis(\"off\")\n            if i == 0:\n                plt.title(cls)\n    plt.show()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Loading & Data Preprocessing\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"(x_train, y_train), (x_valid, y_valid), x_test = load_data(do_reshape=True, resize_shape=(96,96), channel_num=3)\n\nprint(\"Train shape\", x_train.shape, y_train.shape)\nprint(\"Valid shape\", x_valid.shape, y_valid.shape)\nprint(\"Test shape\", x_test.shape)\nshow_imgs(x_train, y_train, rows=4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model Creation:CNN"},{"metadata":{},"cell_type":"markdown","source":"Simple CNN based score: 0.968\n\nImprovements:\n- Data Augmentation +0.01, 0.978\n- Not zero center +0.003, 0.981\n- LearningRateScheduler callbacks + 0.003, 0.983\n- More Dropout +0.002, 0.985\n- Average layers +0.003, 0.988\n- Data Augmentation Enhancement +0.002, 0.99\n- 1x1 kernel +0.001, 0.9916"},{"metadata":{},"cell_type":"markdown","source":"## Implementation 1: Module - Class Instance"},{"metadata":{"trusted":true},"cell_type":"code","source":"class average_CNN(tf.keras.Model):\n    def __init__(self, feature_maps = 32, kernel_sizes = (1,3,5), **kwargs):\n        super(average_CNN, self).__init__(**kwargs)\n        self.cnn_layers = []\n        for kernel_size in kernel_sizes:\n            self.cnn_layers.append(\n                [Conv2D(feature_maps, kernel_size, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\"),\n                Conv2D(feature_maps, kernel_size, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\"),\n                BatchNormalization()\n                ]\n            )\n        self.avg_layer = Average()\n        self.act = Activation('relu')\n        self.bn4 = BatchNormalization()\n        self.pool = MaxPool2D()\n        self.drop = Dropout(0.2)\n        \n    def call(self, inputs):\n        x = inputs\n        self.middle_outs = []\n        for layer_set in self.cnn_layers:\n            for layer in layer_set:\n                x = layer(x)\n            self.middle_outs.append(x)\n            \n        x = self.avg_layer(self.middle_outs)\n        x = self.act(x)\n        x = self.bn4(x)\n        x = self.pool(x)\n        x = self.drop(x)\n        return x\n\nclass my_CNN(tf.keras.Model):\n    def __init__(self, **kwargs):\n        super(my_CNN, self).__init__(**kwargs)\n        self.avg_c1 = average_CNN( 32, (1,3,5))\n        self.avg_c2 = average_CNN( 64, (1,3,5))\n        self.avg_c3 = average_CNN(128, (1,3,5))\n        \n        self.flat = Flatten()\n        self.den1 = Dense(128, activation=\"relu\", kernel_initializer=\"he_uniform\")\n        self.drop = Dropout(0.2)\n        self.den2 = Dense(10, activation=\"softmax\")\n        \n    def call(self, inputs):\n        x = self.avg_c1(inputs)\n        x = self.avg_c2(x)\n        x = self.avg_c3(x)\n        \n        x = self.flat(x)\n        x = self.den1(x)\n        x = self.drop(x)\n        x = self.den2(x)\n        \n        return x\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def module_CNN():\n    my_cnn = my_CNN()\n    my_cnn.compile(loss=\"sparse_categorical_crossentropy\",\n                optimizer=\"adam\",\n                metrics=[\"accuracy\"])\n    return my_cnn","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implementation 2: Functional API"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_CNN():\n    model_input = keras.Input(shape=(28,28,1))\n    \n    C1_1 = Conv2D(32, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(model_input)\n    C1_2 = Conv2D(32, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C1_1)\n    C1_3 = BatchNormalization()(C1_2)\n    C2_1 = Conv2D(32, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(model_input)\n    C2_2 = Conv2D(32, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C2_1)\n    C2_3 = BatchNormalization()(C2_2)\n    C3_1 = Conv2D(32, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(model_input)\n    C3_2 = Conv2D(32, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C3_1)\n    C3_3 = BatchNormalization()(C3_2)\n\n    avg1_1 = Average()([C1_3, C2_3, C3_3])\n    avg1_2 = Activation('relu')(avg1_1)\n    avg1_3 = BatchNormalization()(avg1_2)\n    mp1 = MaxPool2D()(avg1_3)\n    out1 = Dropout(0.2)(mp1)\n    # average1 = average_CNN(feature_maps= 32, kernels = (1, 3, 5))\n    C1_1 = Conv2D(64, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out1)\n    C1_2 = Conv2D(64, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C1_1)\n    C1_3 = BatchNormalization()(C1_2)\n    C2_1 = Conv2D(64, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out1)\n    C2_2 = Conv2D(64, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C2_1)\n    C2_3 = BatchNormalization()(C2_2)\n    C3_1 = Conv2D(64, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out1)\n    C3_2 = Conv2D(64, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C3_1)\n    C3_3 = BatchNormalization()(C3_2)\n\n    avg2_1 = Average()([C1_3, C2_3, C3_3])\n    avg2_2 = Activation('relu')(avg2_1)\n    avg2_3 = BatchNormalization()(avg2_2)\n    mp2 = MaxPool2D()(avg2_3)\n    out2 = Dropout(0.2)(mp2)\n    \n    C1_1 = Conv2D(128, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out2)\n    C1_2 = Conv2D(128, 1, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C1_1)\n    C1_3 = BatchNormalization()(C1_2)\n    C2_1 = Conv2D(128, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out2)\n    C2_2 = Conv2D(128, 3, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C2_1)\n    C2_3 = BatchNormalization()(C2_2)\n    C3_1 = Conv2D(128, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(out2)\n    C3_2 = Conv2D(128, 5, activation=\"relu\", kernel_initializer=\"he_uniform\", padding=\"same\")(C3_1)\n    C3_3 = BatchNormalization()(C3_2)\n\n    avg3_1 = Average()([C1_3, C2_3, C3_3])\n    avg3_2 = Activation('relu')(avg3_1)\n    avg3_3 = BatchNormalization()(avg3_2)\n    mp3 = MaxPool2D()(avg3_3)\n    out3 = Dropout(0.2)(mp3)\n    \n    out = Flatten()(out3)\n    out = Dense(128, activation=\"relu\", kernel_initializer=\"he_uniform\")(out)\n    out = Dropout(0.2)(out)\n    out = Dense(10, activation=\"softmax\")(out)\n    \n    model = tf.keras.Model(inputs=[model_input],outputs=[out])\n    model.compile(loss=\"sparse_categorical_crossentropy\",\n                optimizer=\"adam\",\n                metrics=[\"accuracy\"])\n    return model\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Implementation 3: Transfer Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_MobileNet():\n    base_model = tf.keras.applications.MobileNetV2(weights = 'imagenet', \n                                                   input_shape=(96, 96, 3), \n                                                  include_top = False)\n    base_model.trainable = False\n    \n    for layer in base_model.layers:        \n        print(\"layer:\", layer.name, \", trainable:\",layer.trainable)\n        \n    x = GlobalAveragePooling2D()(base_model.output)\n    x = Dropout(0.4)(x)\n    output = Dense(10, activation=\"softmax\")(x)\n    model = keras.Model(inputs=base_model.input, outputs = output)\n    model.compile(loss=\"sparse_categorical_crossentropy\",\n                optimizer=\"adam\",\n                metrics=[\"accuracy\"])\n    \n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data augmentations Type1: preprocessing layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nrescale_layer = tf.keras.Sequential([layers.experimental.preprocessing.Rescaling(1./255)])\n\n## Example from https://www.tensorflow.org/tutorials/images/data_augmentation\nresize_and_rescale = tf.keras.Sequential([\n  layers.experimental.preprocessing.Rescaling(1./255)\n])\n\ndata_augmentation = tf.keras.Sequential([\n  # Contrast\n  layers.experimental.preprocessing.RandomContrast(0.5),\n  # Rotate\n  layers.experimental.preprocessing.RandomRotation(factor = (0.2), seed=42),\n  # Zoom\n  layers.experimental.preprocessing.RandomZoom(0.1),\n  # Scaling\n  #layers.experimental.preprocessing.Rescaling(1./255),\n  #layers.experimental.preprocessing.Rescaling(1./127.5, offset=-1),\n])\n\nbatch_size = 32\nAUTOTUNE = -1\n\ndef prepare(ds, shuffle=False, augment=False):\n  # Resize and rescale all datasets\n  #ds = ds.map(lambda x, y: (resize_and_rescale(x), y))\n  x = resize_and_rescale(x)  \n\n  if shuffle:\n    ds = ds.shuffle(1000)\n\n  # Batch all datasets\n  ds = ds.batch(batch_size)\n\n  # Use data augmentation only on the training set\n  if augment:\n    #ds = ds.map(lambda x, y: (data_augmentation(x, training=True), y))\n    ds = data_augmentation(x, training=True)\n  # Use buffered prefecting on all datasets\n  return ds.prefetch(-1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data augmentations Type2: ImageDataGenerator\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\ndatagen = ImageDataGenerator(rescale=1/255.,\n                             zoom_range = 0.3,\n                             height_shift_range = 0.3,\n                            width_shift_range = 0.3,\n                            rotation_range = 25,\n                            shear_range=0.5,\n                            fill_mode=\"constant\",\n                            cval=0\n                            )\nvalid_datagen = ImageDataGenerator(rescale=1./255)\ntest_datagen = ImageDataGenerator(rescale=1./255)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"效果不錯:\n\nzoom_range = 0.2,height_shift_range = 0.2,width_shift_range = 0.2,rotation_range = 20,shear_range=0.4\n\nzoom_range = 0.1,height_shift_range = 0.1,width_shift_range = 0.1,rotation_range = 10,shear_range=0.2\n"},{"metadata":{},"cell_type":"markdown","source":"### Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def summarize_diagnostics(history):\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['training', 'validation'])\n    plt.figure()\n\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['loss', 'validation loss'])\n    plt.figure()\n    \n    plt.show()\n\nfrom keras.callbacks import LearningRateScheduler, EarlyStopping\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)\nearlystop = EarlyStopping(patience = 15)\n\n# Version a: Implement in module\ndef do_module_CNN(x_train, y_train, x_valid, y_valid):\n    model_cnn = module_CNN()\n    history = model_cnn.fit_generator(datagen.flow(x_train, y_train, batch_size=64),\n                                      steps_per_epoch= x_train.shape[0] // 64,\n                                      epochs=100,verbose=2,\n                                      validation_data=valid_datagen.flow(x_valid, y_valid, batch_size=64),\n                                      validation_steps= x_valid.shape[0] // 64,\n                                      callbacks=[annealer, earlystop])\n\n    return model_cnn, history\n\n#Version b: Implement in function call\ndef do_CNN(x_train, y_train, x_valid, y_valid):\n    model_cnn = model_CNN()\n    history = model_cnn.fit_generator(datagen.flow(x_train, y_train, batch_size=64),\n                                      steps_per_epoch= x_train.shape[0] // 64,\n                                      epochs=100,verbose=2,\n                                      validation_data=valid_datagen.flow(x_valid, y_valid, batch_size=64),\n                                      validation_steps= x_valid.shape[0] // 64,\n                                      callbacks=[annealer, earlystop])\n\n    return model_cnn, history\n\ndef do_MobileNet(x_train, y_train, x_valid, y_valid):\n    model_mbn = model_MobileNet()\n    history1 = model_mbn.fit_generator(datagen.flow(x_train, y_train, batch_size=64),\n                                      steps_per_epoch= x_train.shape[0] // 64,\n                                      epochs=100,verbose=2,\n                                      validation_data=valid_datagen.flow(x_valid, y_valid, batch_size=64),\n                                      validation_steps= x_valid.shape[0] // 64,\n                                      callbacks=[annealer, earlystop])\n    print(\"=========Phase 2: Unfreeze lower layers========\")\n    model_mbn.trainable=True\n    for layer in model_mbn.layers:        \n        print(\"layer:\", layer.name, \", trainable:\",layer.trainable)\n    model_mbn.compile(loss=\"sparse_categorical_crossentropy\",\n                optimizer=\"adam\",\n                metrics=[\"accuracy\"])\n    history2 = model_mbn.fit_generator(datagen.flow(x_train, y_train, batch_size=64),\n                                      steps_per_epoch= x_train.shape[0] // 64,\n                                      epochs=100,verbose=2,\n                                      validation_data=valid_datagen.flow(x_valid, y_valid, batch_size=64),\n                                      validation_steps= x_valid.shape[0] // 64,\n                                      callbacks=[annealer, earlystop])    \n    return model_mbn, history2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\nstart_time = time.time()\nmodel, history = do_MobileNet(x_train, y_train, x_valid, y_valid)\nprint(\"--- Model Train: %s seconds ---\" % (time.time() - start_time))\n\nsummarize_diagnostics(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## End: Predict Test set "},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\n\n#y_hat = model.predict(x_test, batch_size=64)\ny_hat = model.predict_generator(test_datagen.flow(x_test, shuffle=False))\ny_pred = np.argmax(y_hat,axis=1)\n\nprint(\"--- Model Predict: %s seconds ---\" % (time.time() - start_time))\nprint(\"y shape\", y_pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_submit=pd.DataFrame({\"id\": list(range(len(y_pred))),\"label\": y_pred})\npd_submit.to_csv('submission.csv',index=False,header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd_submit['label'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}