{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport PIL\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score\n\nfrom tensorflow.keras import datasets, layers, models\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom keras.utils.np_utils import to_categorical\nfrom keras.preprocessing.image import ImageDataGenerator\n","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:07.802635Z","iopub.execute_input":"2023-07-30T14:35:07.803219Z","iopub.status.idle":"2023-07-30T14:35:07.812979Z","shell.execute_reply.started":"2023-07-30T14:35:07.803176Z","shell.execute_reply":"2023-07-30T14:35:07.811928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Get Data","metadata":{}},{"cell_type":"code","source":"def get_data(data_set, label_column='label'):\n    # data_set:   pd DataFrame\n    data_X = data_set.drop(columns=label_column)\n    data_y = data_set[label_column]\n    return data_X, data_y","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:07.815055Z","iopub.execute_input":"2023-07-30T14:35:07.815773Z","iopub.status.idle":"2023-07-30T14:35:07.833866Z","shell.execute_reply.started":"2023-07-30T14:35:07.815725Z","shell.execute_reply":"2023-07-30T14:35:07.832659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# read data\ntrain_set = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntest_set = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\ndig_set = pd.read_csv('/kaggle/input/Kannada-MNIST/Dig-MNIST.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:07.836226Z","iopub.execute_input":"2023-07-30T14:35:07.836681Z","iopub.status.idle":"2023-07-30T14:35:15.899675Z","shell.execute_reply.started":"2023-07-30T14:35:07.836640Z","shell.execute_reply":"2023-07-30T14:35:15.898434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train = get_data(train_set, label_column='label')\nX_dig, y_dig = get_data(dig_set, label_column='label')","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:15.901325Z","iopub.execute_input":"2023-07-30T14:35:15.901881Z","iopub.status.idle":"2023-07-30T14:35:16.120890Z","shell.execute_reply.started":"2023-07-30T14:35:15.901829Z","shell.execute_reply":"2023-07-30T14:35:16.119692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape, y_train.shape, X_dig.shape, y_dig.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.122465Z","iopub.execute_input":"2023-07-30T14:35:16.122847Z","iopub.status.idle":"2023-07-30T14:35:16.131062Z","shell.execute_reply.started":"2023-07-30T14:35:16.122813Z","shell.execute_reply":"2023-07-30T14:35:16.130050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.to_numpy().reshape(-1, 28, 28, 1)\ny_train = y_train.to_numpy().reshape(-1, 1)\nX_dig = X_dig.to_numpy().reshape(-1, 28, 28, 1)\ny_dig = y_dig.to_numpy().reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.135901Z","iopub.execute_input":"2023-07-30T14:35:16.136275Z","iopub.status.idle":"2023-07-30T14:35:16.148030Z","shell.execute_reply.started":"2023-07-30T14:35:16.136239Z","shell.execute_reply":"2023-07-30T14:35:16.146879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = test_set.drop(columns='id').to_numpy().reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.150199Z","iopub.execute_input":"2023-07-30T14:35:16.150608Z","iopub.status.idle":"2023-07-30T14:35:16.170872Z","shell.execute_reply.started":"2023-07-30T14:35:16.150562Z","shell.execute_reply":"2023-07-30T14:35:16.169757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# no need to do this for CNN, otherwise some image preprocessing packages \n# can convert any value between 0-1 to 0 or 1\n# X_train = X_train / 255.0\n# X_dig = X_dig / 255.0\n# test_set = test_set / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.172268Z","iopub.execute_input":"2023-07-30T14:35:16.172636Z","iopub.status.idle":"2023-07-30T14:35:16.185228Z","shell.execute_reply.started":"2023-07-30T14:35:16.172596Z","shell.execute_reply":"2023-07-30T14:35:16.184141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_cat = to_categorical(y_train)\ny_dig_cat = to_categorical(y_dig)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.186657Z","iopub.execute_input":"2023-07-30T14:35:16.187088Z","iopub.status.idle":"2023-07-30T14:35:16.205488Z","shell.execute_reply.started":"2023-07-30T14:35:16.187028Z","shell.execute_reply":"2023-07-30T14:35:16.204436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nclass_names = np.arange(10)\n\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_train[i,:,:,0], cmap='gray')\n    # y_train_cat is in one-hot format\n    plt.xlabel(np.where(y_train_cat[i]==1)[0][0])\nplt.show()\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.207100Z","iopub.execute_input":"2023-07-30T14:35:16.207728Z","iopub.status.idle":"2023-07-30T14:35:16.219111Z","shell.execute_reply.started":"2023-07-30T14:35:16.207668Z","shell.execute_reply":"2023-07-30T14:35:16.218078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. array to rgb","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def array2rgb(input_array, reshape_size, interp_method):\n    output_list = []\n    for i in range(len(input_array)):\n        data = input_array[i,:,:, 0]\n        img_rgb = Image.fromarray(data.astype('uint8')).convert('RGB')\n        img_resized = img_rgb.resize(reshape_size, resample=interp_method)\n        img_resized = np.array(img_resized).astype('float32')\n        output_list.append(img_resized)\n    output_array = np.array(output_list)\n\n    return output_array","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.220821Z","iopub.execute_input":"2023-07-30T14:35:16.221397Z","iopub.status.idle":"2023-07-30T14:35:16.237454Z","shell.execute_reply.started":"2023-07-30T14:35:16.221360Z","shell.execute_reply":"2023-07-30T14:35:16.236104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reshape_size = (94, 94)\ninterp_method = PIL.Image.BICUBIC\n\nX_train_rgb = array2rgb(X_train, reshape_size=reshape_size, interp_method=interp_method)\nX_dig_rgb = array2rgb(X_dig, reshape_size=reshape_size, interp_method=interp_method)\ntest_set_rgb = array2rgb(test_set, reshape_size=reshape_size, interp_method=interp_method)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:16.239004Z","iopub.execute_input":"2023-07-30T14:35:16.239658Z","iopub.status.idle":"2023-07-30T14:35:42.532945Z","shell.execute_reply.started":"2023-07-30T14:35:16.239607Z","shell.execute_reply":"2023-07-30T14:35:42.531775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train_rgb.shape, X_dig_rgb.shape, test_set_rgb.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:42.534382Z","iopub.execute_input":"2023-07-30T14:35:42.534767Z","iopub.status.idle":"2023-07-30T14:35:42.540023Z","shell.execute_reply.started":"2023-07-30T14:35:42.534731Z","shell.execute_reply":"2023-07-30T14:35:42.539235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nclass_names = np.arange(10)\n\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X_train_rgb[i,:,:,:], cmap='gray')\n    # y_train_cat is in one-hot format\n    plt.xlabel(np.where(y_train_cat[i]==1)[0][0])\nplt.show()\n\"\"\";","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:42.541213Z","iopub.execute_input":"2023-07-30T14:35:42.541703Z","iopub.status.idle":"2023-07-30T14:35:42.556628Z","shell.execute_reply.started":"2023-07-30T14:35:42.541659Z","shell.execute_reply":"2023-07-30T14:35:42.555644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. data generator","metadata":{}},{"cell_type":"code","source":"height_shift_range = 0.1\nwidth_shift_range = 0.1\nzoom_range = 0.1\nrotation_range=10\ndatagen = ImageDataGenerator(rotation_range=10, zoom_range=zoom_range,\n                             width_shift_range=width_shift_range,\n                             height_shift_range=height_shift_range)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:42.558237Z","iopub.execute_input":"2023-07-30T14:35:42.558611Z","iopub.status.idle":"2023-07-30T14:35:42.572843Z","shell.execute_reply.started":"2023-07-30T14:35:42.558574Z","shell.execute_reply":"2023-07-30T14:35:42.571856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nit = datagen.flow(X_train_rgb[0:10,:,:,:], batch_size=1)\nplt.figure(figsize=(5, 5))\nfor i in range(9):\n    plt.subplot(330 + 1+ i)\n    batch = it.next()\n    image = batch[0].astype('uint8')\n    #print(image.shape)\n    plt.imshow(image, cmap='gray')\n\n\"\"\";","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:42.574294Z","iopub.execute_input":"2023-07-30T14:35:42.574667Z","iopub.status.idle":"2023-07-30T14:35:42.585934Z","shell.execute_reply.started":"2023-07-30T14:35:42.574633Z","shell.execute_reply":"2023-07-30T14:35:42.585011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. build model","metadata":{}},{"cell_type":"code","source":"base_model = tf.keras.applications.mobilenet_v2.MobileNetV2(weights = 'imagenet', input_shape = (94, 94, 3), include_top = False)\n\nbase_model.trainable =False","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:42.587203Z","iopub.execute_input":"2023-07-30T14:35:42.587689Z","iopub.status.idle":"2023-07-30T14:35:45.399466Z","shell.execute_reply.started":"2023-07-30T14:35:42.587637Z","shell.execute_reply":"2023-07-30T14:35:45.398612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = models.Sequential([\n    base_model, layers.GlobalAveragePooling2D()\n    ,\n    layers.Dropout(0.4), layers.Dense(10, activation='softmax')\n])\n\n\"\"\"\n# Another way to implement the model\ninputs = tf.keras.Input(shape=(94, 94, 3))\nx = base_model(inputs)\nx = tf.keras.layers.GlobalAveragePooling2D()(x)\nx = layers.Dropout(0.4)(x)\noutputs = layers.Dense(10, activation='softmax')(x)\nmodel = tf.keras.Model(inputs, outputs)\n\"\"\"\nprint()","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:45.400831Z","iopub.execute_input":"2023-07-30T14:35:45.401158Z","iopub.status.idle":"2023-07-30T14:35:45.755820Z","shell.execute_reply.started":"2023-07-30T14:35:45.401126Z","shell.execute_reply":"2023-07-30T14:35:45.754599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adam optimizer\nlearning_rate = 0.001\nbeta_1 = 0.9\nbeta_2 = 0.999\noptimizer = Adam(lr=learning_rate, beta_1=beta_1, beta_2=beta_2)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:45.757345Z","iopub.execute_input":"2023-07-30T14:35:45.757741Z","iopub.status.idle":"2023-07-30T14:35:45.763564Z","shell.execute_reply.started":"2023-07-30T14:35:45.757705Z","shell.execute_reply":"2023-07-30T14:35:45.762364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=optimizer,\n              loss=['categorical_crossentropy'],\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:45.764863Z","iopub.execute_input":"2023-07-30T14:35:45.765213Z","iopub.status.idle":"2023-07-30T14:35:45.797583Z","shell.execute_reply.started":"2023-07-30T14:35:45.765180Z","shell.execute_reply":"2023-07-30T14:35:45.796345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 32\n# reduce learning rate\nreduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.5,\n                              patience=3, min_lr=0.00001)\nhistory = model.fit(datagen.flow(X_train_rgb, y_train_cat, batch_size=batch_size), callbacks=[reduce_lr],\n          steps_per_epoch=len(X_train) / batch_size, epochs=10, validation_data=(X_dig_rgb, y_dig_cat))","metadata":{"execution":{"iopub.status.busy":"2023-07-30T14:35:45.799138Z","iopub.execute_input":"2023-07-30T14:35:45.799520Z","iopub.status.idle":"2023-07-30T15:24:37.943491Z","shell.execute_reply.started":"2023-07-30T14:35:45.799461Z","shell.execute_reply":"2023-07-30T15:24:37.942607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history.history","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:24:37.945228Z","iopub.execute_input":"2023-07-30T15:24:37.945572Z","iopub.status.idle":"2023-07-30T15:24:37.950917Z","shell.execute_reply.started":"2023-07-30T15:24:37.945539Z","shell.execute_reply":"2023-07-30T15:24:37.949836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nplt.plot(history.history['acc'], label='accuracy')\nplt.plot(history.history['val_acc'], label = 'val_accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(loc='lower right')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:24:37.952533Z","iopub.execute_input":"2023-07-30T15:24:37.952900Z","iopub.status.idle":"2023-07-30T15:24:37.969011Z","shell.execute_reply.started":"2023-07-30T15:24:37.952865Z","shell.execute_reply":"2023-07-30T15:24:37.967920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start= time.time()\ny_pred = model.predict_classes(test_set_rgb)\nend = time.time()\ntest_time = (end - start) / 60\nprint(f'testing time: {test_time}')","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:24:37.970758Z","iopub.execute_input":"2023-07-30T15:24:37.971092Z","iopub.status.idle":"2023-07-30T15:24:55.264301Z","shell.execute_reply.started":"2023-07-30T15:24:37.971060Z","shell.execute_reply":"2023-07-30T15:24:55.262585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_pd = pd.DataFrame({'id': np.arange(len(y_pred)),\n                          'label': y_pred})\ny_pred_pd.to_csv(\"submission.csv\", sep=',', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:24:55.266197Z","iopub.execute_input":"2023-07-30T15:24:55.266984Z","iopub.status.idle":"2023-07-30T15:24:55.697912Z","shell.execute_reply.started":"2023-07-30T15:24:55.266931Z","shell.execute_reply":"2023-07-30T15:24:55.697017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. compare to the previous model","metadata":{}},{"cell_type":"code","source":"\"\"\"\n# conv layers\nmodel_2 = models.Sequential()\nmodel_2.add(layers.Conv2D(32, kernel_size=(3, 3),padding='Same', activation='relu', input_shape=(28, 28, 1)))\nmodel_2.add(layers.MaxPooling2D((2, 2)))\nmodel_2.add(layers.Conv2D(32, kernel_size=(3, 3), padding='Same', activation='relu'))\nmodel_2.add(layers.BatchNormalization(momentum=0.1))\nmodel_2.add(layers.MaxPooling2D((2, 2)))\nmodel_2.add(layers.Dropout(0.2))\n\nmodel_2.add(layers.Conv2D(64, kernel_size=(5, 5),padding='Same', activation='relu', input_shape=(28, 28, 1)))\nmodel_2.add(layers.MaxPooling2D((2, 2)))\nmodel_2.add(layers.Conv2D(64, kernel_size=(5, 5), padding='Same', activation='relu'))\nmodel_2.add(layers.BatchNormalization(momentum=0.1))\nmodel_2.add(layers.MaxPooling2D((2, 2)))\nmodel_2.add(layers.Dropout(0.2))\n\n# Fully connected layers\nmodel_2.add(layers.Flatten())\nmodel_2.add(layers.Dense(128, activation='relu'))\nmodel_2.add(layers.Dropout(0.4))\nmodel_2.add(layers.Dense(64, activation='relu'))\nmodel_2.add(layers.Dropout(0.4))\nmodel_2.add(layers.Dense(10, activation='softmax'))\n\nmodel_2.compile(optimizer=optimizer,\n              loss=['categorical_crossentropy'],\n              metrics=['accuracy'])\n\n\nhistory_2 = model_2.fit(datagen.flow(X_train, y_train_cat, batch_size=batch_size), callbacks=[reduce_lr],\n          steps_per_epoch=len(X_train) / batch_size, epochs=10, validation_data=(X_dig, y_dig_cat))\n\nfig, axs = plt.subplots(1, 2, figsize=(10,5))\naxs[0].plot(history_2.history['acc'], label='accuracy')\naxs[0].plot(history_2.history['val_acc'], label = 'val_accuracy')\naxs[0].set_ylim(0.5, 1)\naxs[0].set_title('transfer learning: MobileNetV2')\naxs[0].set_xlabel('Epoch')\naxs[0].set_ylabel('Accuracy')\naxs[0].legend(loc='lower right')\n\naxs[1].plot(history.history['acc'], label='accuracy')\naxs[1].plot(history.history['val_acc'], label = 'val_accuracy')\naxs[1].set_ylim(0.5, 1)\naxs[1].set_title('classic CNN')\naxs[1].set_xlabel('Epoch')\naxs[1].set_ylabel('Accuracy')\naxs[1].legend(loc='lower right')\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-07-30T15:24:55.699452Z","iopub.execute_input":"2023-07-30T15:24:55.700104Z","iopub.status.idle":"2023-07-30T15:24:55.710340Z","shell.execute_reply.started":"2023-07-30T15:24:55.700057Z","shell.execute_reply":"2023-07-30T15:24:55.709030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It seems that the classic CNN model works better, we might need to unfreeze the weights of the MobileNetV2 model to achieve similar or better result.","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}