{"cells":[{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder \nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Data path\ntrain_set = \"../input/Kannada-MNIST/train.csv\"\nvalid_set = \"../input/Kannada-MNIST/Dig-MNIST.csv\"\ntest_set = \"../input/Kannada-MNIST/test.csv\"\n\n# Read data\ntrain_csv = pd.read_csv(train_set)\nvalid_csv = pd.read_csv(valid_set)\ntest_csv = pd.read_csv(test_set)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def image_generator(images_csv, lbl=True):\n    labels = 0\n    if (lbl==True):\n        labels = images_csv[images_csv.columns[0]].to_numpy(dtype=np.float64, copy=True)\n\n        labels = tf.keras.utils.to_categorical(labels, num_classes=10)\n\n    images = images_csv.loc[:,'pixel0':'pixel783'].to_numpy(dtype=np.float64, copy=True)\n    # Reshape 28x28x1\n    images = images.reshape((len(images),28,28,1))\n    # Normalization\n    images = images / 255.\n    return images, labels","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images, train_labels = image_generator(train_csv)\nvalid_images, valid_labels = image_generator(valid_csv)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, y_train, X_test, y_test = train_test_split(np.concatenate((train_images, valid_images)),\n                                                   np.concatenate((train_labels, valid_labels)),\n                                                   test_size=0.1,\n                                                   shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Construct my model"},{"metadata":{},"cell_type":"markdown","source":"- Main idea: My model use average CNN blocks combined from three Conv2D+BN layers, averaged block and a CNN block.\n- Note: \n + I chose 3 Conv2D with respect to three kernel (1,1), (3,3), (5,5). With (1,1) kernel, it do nothing affect on the image size but (3,3) and (5,5) do, so I get padding=\"same\" to keep all feature maps in the same shape.\n + The kernel size of CNN blocks will be decreased depend on the feature map size to help the model can make it through all average CNN blocks.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def cnn_model():\n    inp = tf.keras.Input(shape=(28,28,1))\n    x1 = tf.keras.layers.Conv2D(128, (1,1), strides=(1,1), activation='relu')(inp)\n    x1 = tf.keras.layers.BatchNormalization()(x1)\n    x3 = tf.keras.layers.Conv2D(128, (3,3), padding='same', strides=(1,1), activation='relu')(inp)\n    x3 = tf.keras.layers.BatchNormalization()(x3)\n    x5 = tf.keras.layers.Conv2D(128, (5,5), padding='same', strides=(1,1), activation='relu')(inp)\n    x5 = tf.keras.layers.BatchNormalization()(x5)\n\n    averaged = tf.keras.layers.Average()([x1,x3,x5])\n    averaged = tf.keras.layers.Activation('relu')(averaged)\n    averaged = tf.keras.layers.BatchNormalization()(averaged)\n\n    x = tf.keras.layers.Conv2D(128, (5,5), strides=(1,1), activation='relu')(averaged)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    x = tf.keras.layers.Dropout(rate=0.25)(x)\n\n\n    x1 = tf.keras.layers.Conv2D(128, (1,1), strides=(1,1), activation='relu')(x)\n    x1 = tf.keras.layers.BatchNormalization()(x1)\n    x3 = tf.keras.layers.Conv2D(128, (3,3), padding='same', strides=(1,1), activation='relu')(x)\n    x3 = tf.keras.layers.BatchNormalization()(x3)\n    x5 = tf.keras.layers.Conv2D(128, (5,5), padding='same', strides=(1,1), activation='relu')(x)\n    x5 = tf.keras.layers.BatchNormalization()(x5)\n\n    averaged = tf.keras.layers.Average()([x1,x3,x5])\n    averaged = tf.keras.layers.Activation('relu')(averaged)\n    averaged = tf.keras.layers.BatchNormalization()(averaged)\n\n    x = tf.keras.layers.Conv2D(128, (3,3), strides=(1,1), activation='relu')(averaged)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    x = tf.keras.layers.Dropout(rate=0.25)(x)\n\n    x1 = tf.keras.layers.Conv2D(256, (1,1), strides=(1,1), activation='relu')(x)\n    x1 = tf.keras.layers.BatchNormalization()(x1)\n    x3 = tf.keras.layers.Conv2D(256, (3,3), padding='same', strides=(1,1), activation='relu')(x)\n    x3 = tf.keras.layers.BatchNormalization()(x3)\n    x5 = tf.keras.layers.Conv2D(256, (5,5), padding='same', strides=(1,1), activation='relu')(x)\n    x5 = tf.keras.layers.BatchNormalization()(x5)\n\n    averaged = tf.keras.layers.Average()([x1,x3,x5])\n    averaged = tf.keras.layers.Activation('relu')(averaged)\n    averaged = tf.keras.layers.BatchNormalization()(averaged)\n\n    x = tf.keras.layers.Conv2D(256, (1,1), strides=(1,1), activation='relu')(averaged)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    x = tf.keras.layers.Dropout(rate=0.25)(x)\n\n    x1 = tf.keras.layers.Conv2D(256, (1,1), strides=(1,1), activation='relu')(x)\n    x1 = tf.keras.layers.BatchNormalization()(x1)\n    x3 = tf.keras.layers.Conv2D(256, (3,3), padding='same', strides=(1,1), activation='relu')(x)\n    x3 = tf.keras.layers.BatchNormalization()(x3)\n    x5 = tf.keras.layers.Conv2D(256, (5,5), padding='same', strides=(1,1), activation='relu')(x)\n    x5 = tf.keras.layers.BatchNormalization()(x5)\n\n    averaged = tf.keras.layers.Average()([x1,x3,x5])\n    averaged = tf.keras.layers.Activation('relu')(averaged)\n    averaged = tf.keras.layers.BatchNormalization()(averaged)\n\n    x = tf.keras.layers.Conv2D(256, (1,1), strides=(1,1), activation='relu')(averaged)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.MaxPooling2D()(x)\n    x = tf.keras.layers.Dropout(rate=0.25)(x)\n    \n    x = tf.keras.layers.Flatten()(x)\n    x = tf.keras.layers.Dense(256, activation=tf.nn.relu)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dense(128, activation=tf.nn.relu)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n    x = tf.keras.layers.Dense(64, activation=tf.nn.relu)(x)\n    x = tf.keras.layers.BatchNormalization()(x)\n\n    \n    output = tf.keras.layers.Dense(10, activation=tf.nn.softmax)(x)\n\n    model = tf.keras.Model(inputs=inp, outputs=output)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = cnn_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training strategy"},{"metadata":{},"cell_type":"markdown","source":"- I train in Google Colab and load model weights as dataset in Kaggle.\n- The trainning process use Data Augmentation, all range values I set from 10 to 30 (increase when my model had trained well)\n- With test images, I let the model predicts test labels and take all of them into the trainning dataset, do this again and again to ensure my model get the final prediction on test images."},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights(\"../input/pretrained/model2.h5\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 1\nBATCH_SIZE = 128\nlr = 0.00000001\nopt = tf.keras.optimizers.Adam(learning_rate=lr)\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_loss', \n                                            patience=10,\n                                            verbose=1,\n                                            factor=0.75)\nes = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_aug = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range=10,\n                                                  width_shift_range=0.2,\n                                                  height_shift_range=0.2,\n                                                  shear_range=0.1,\n                                                  zoom_range=0.2,\n                                                  horizontal_flip=False)\nvalid_aug = tf.keras.preprocessing.image.ImageDataGenerator()\n\nmcp = tf.keras.callbacks.ModelCheckpoint(\"../input/pretrained-models/model.h5\", monitor='val_loss',\n                                             save_best_only=True, save_weights_only=True)\nmodel.compile(optimizer=opt, loss=tf.keras.losses.CategoricalCrossentropy(),metrics=['accuracy'])\nmodel.fit_generator(train_aug.flow(train_images, train_labels,batch_size=BATCH_SIZE),\n                    steps_per_epoch=10,\n                    validation_data=valid_aug.flow(valid_images, valid_labels),\n                    validation_steps=50,\n                    epochs=EPOCHS, verbose=1,\n                       callbacks=[learning_rate_reduction, es])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_images, _ = image_generator(test_csv, lbl=False)\n\npreds = model.predict(test_images)\npreds = preds.argmax(axis=1)\nprint(preds)\n# predictions to dataframe\npreds = preds.astype(int).flatten()\npreds = (LabelEncoder().fit_transform((preds)))\npreds = pd.DataFrame({'label': preds})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(data=test_csv.id)\nsub = sub.join(preds)\n# Write out the predictions to disk\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"That's all! \n\nYou must adjust all the hyperparameters more suitable for your model because all of them above were just a temporary values to make my commit run as fast as possible.\n\n### *If you find my notebook help you in someways, please upvote it! Thanks you!.*"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}