{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"##### This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt  \n\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, Dropout, Dense, Flatten, DepthwiseConv2D, MaxPooling2D\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint\nfrom tensorflow.keras.activations import relu\n\nfrom sklearn.model_selection import train_test_split\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define constants\nBASE_PATH = '/kaggle/input/Kannada-MNIST/'\n\n# set random seed\nnp.random.seed(1973)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load data files \n\nLoading datase by pandas dataframe.\n\n- train.csv : for Training data \n- test.csv : for Test data \n- Dig-MNIST.csv : additional labeled data. This file can use for validation and test model. very kindly data for me (us?).\n- sample_submission.csv : sample data for submission."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_src_df = pd.read_csv(filepath_or_buffer=BASE_PATH+'train.csv')\ntest_src_df = pd.read_csv(filepath_or_buffer=BASE_PATH+'test.csv')\ndm_src_df = pd.read_csv(filepath_or_buffer=BASE_PATH+'Dig-MNIST.csv')\nsubmission = pd.read_csv(filepath_or_buffer=BASE_PATH+'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### train.csv \ntrain.csv contain 60,000 labeled data. each data have label and 1d pixcel data.  \nPixcel data may be necessary convet to 2d pixcel data.  \nPixel data takes a value in the range of 0-255.  \nLabel data takes a value in the range of 0-9.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_src_df.info())\nprint(train_src_df.head())\nprint(train_src_df.describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### test.csv  \ntest.csv has 5,000 non labeled data.  \nfirst column is for id. The second and subsequent columns are pixel data.  \ndata range is same as train.csv."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_src_df.info())\nprint(test_src_df.head())\nprint(test_src_df.describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dig-MNIST  \nDig-MNIST.csv has 10,240 labeled data.  \nIt's enough values for validation and test."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(dm_src_df.info())\nprint(dm_src_df.head())\nprint(dm_src_df.describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### preprocess  \nseparate label and pixcel data. and, convert shape of pixcel data, 1d to 3d (height , width, channel).  \nand create ImageDataGenerator for data augumentation, for avoid overfitting. "},{"metadata":{"trusted":true},"cell_type":"code","source":"all_data = pd.concat([train_src_df, dm_src_df])\n'''\nX_train = train_src_df.iloc[:, 1:].to_numpy().reshape(-1, 28, 28, 1)\ny_train = to_categorical(train_src_df.iloc[:, 0])\nX_additional = dm_src_df.iloc[:, 1:].to_numpy().reshape(-1, 28, 28, 1)\ny_additional = to_categorical(dm_src_df.iloc[:, 0])\nX_valid, X_test, y_valid, y_test = train_test_split(X_additional, y_additional, test_size=0.5, shuffle=True)\n'''\nX_all_data = all_data.iloc[:, 1:].to_numpy().reshape(-1, 28, 28, 1)\ny_all_data = to_categorical(all_data.iloc[:, 0])\n\nX_train, X_valid, y_train, y_valid = train_test_split(X_all_data, y_all_data, test_size=0.07, shuffle=True)\ntest_data = test_src_df.iloc[:, 1:].to_numpy().reshape(-1, 28, 28, 1)\nprint(X_train.shape)\nprint(y_train.shape)\nprint(X_valid.shape)\nprint(y_valid.shape)\nprint(test_data.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(3, 3, figsize=(15,15))\nfor idx in range(9):\n    i = idx % 3 # Get subplot row\n    j = idx // 3 # Get subplot column\n    axes[i, j].imshow(X_train[idx].reshape(28, 28))\n\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen = ImageDataGenerator(\n    featurewise_center=False,\n    samplewise_center=False,\n    featurewise_std_normalization=False,\n    samplewise_std_normalization=False,\n    zca_whitening=False,\n    zca_epsilon=1e-06,\n    rotation_range=15,\n    width_shift_range=0.2,\n    height_shift_range=0.2,\n    brightness_range=[0.7,1.0],\n    shear_range=0.2,\n    zoom_range=0.2, \n    channel_shift_range=0.0, \n    fill_mode='constant', \n    cval=0.0, \n    horizontal_flip=False, \n    vertical_flip=False, \n    rescale=1./255, \n    preprocessing_function=None, \n    data_format='channels_last', \n    validation_split=0.0, \n    dtype='float32')\n\ntest_datagen = ImageDataGenerator(rescale=1./255)\n\ntrain_datagen.fit(X_train)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define model \ndefine model like vgg 16. \nI use activation function, 'gelu', used in BERT.  \n... but, gelu did not perform as well as relu. The function may be a bit too complex.\nfollowing code is refered from follwoing web page.  \n[gelu in BERT](https://github.com/google-research/bert/blob/bee6030e31e42a9394ac567da170a89a98d2062f/modeling.py#L264)  \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def gelu(x):\n    \"\"\"Gaussian Error Linear Unit.\n    This is a smoother version of the RELU.\n    Original paper: https://arxiv.org/abs/1606.08415\n    refer : https://github.com/google-research/bert/blob/bee6030e31e42a9394ac567da170a89a98d2062f/modeling.py#L264\n    Args:\n        x: float Tensor to perform activation.\n    Returns:\n        `x` with the GELU activation applied.\n    \"\"\"\n    cdf = 0.5 * (1.0 + tf.tanh(\n        (np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))\n    return x * cdf\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(fig_size=28, channel_num=1, class_num=10, activation=relu):\n    model = Sequential()\n    \n    # 28*2 -> 14\n    model.add(Conv2D(64, (5, 5), padding='same', activation=activation, input_shape=(fig_size, fig_size, channel_num)))\n    model.add(Dropout(0.5))\n    model.add(Conv2D(64, (5, 5), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(MaxPooling2D(2))\n\n    # 14*2 -> 7\n    model.add(Conv2D(128, (3, 3), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Conv2D(128, (3, 3), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(MaxPooling2D(2))\n\n    # 7*3 -> 3\n    model.add(Conv2D(256, (3, 3), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Conv2D(256, (3, 3), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Conv2D(256, (3, 3), padding='same', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(MaxPooling2D(2))\n\n    # 3 -> 1 -> flatten\n    model.add(Conv2D(1024, (1, 1), activation=activation))\n    model.add(Dropout(0.5))\n    model.add(DepthwiseConv2D((3, 3), padding='valid', activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Flatten())\n    \n    # fully connection and classify\n    model.add(Dense(512, activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Dense(512, activation=activation))\n    model.add(Dropout(0.5))\n    model.add(Dense(class_num, activation='softmax'))\n\n    # compile\n    model.compile(optimizer=Adam(),\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Fit and evaluate \nfit and evaluate both relu and gelu, and compare validation loss and acc."},{"metadata":{"trusted":true},"cell_type":"code","source":"model_relu = create_model(activation=relu)\nmodel_gelu = create_model(activation=gelu)\nmodel_relu.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define callbacks\nrlr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=3, min_lr=1.0e-8, verbose=1)\nmcp = ModelCheckpoint(filepath='weights.hdf5', verbose=1, save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define training parameters\nBATCH_SIZE = 64\nEPOCH_NUM = 30","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#training relu\nhistory_relu = model_relu.fit_generator(\n        train_datagen.flow(X_train, y_train, batch_size=BATCH_SIZE) , \n        validation_data=test_datagen.flow(X_valid, y_valid, batch_size=BATCH_SIZE), \n        epochs=EPOCH_NUM, \n        callbacks=[rlr, mcp]\n        )\nmodel_relu.load_weights('weights.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#training gelu\nhistory_gelu = model_gelu.fit_generator(\n        train_datagen.flow(X_train, y_train, batch_size=BATCH_SIZE) , \n        validation_data=test_datagen.flow(X_valid, y_valid, batch_size=BATCH_SIZE), \n        epochs=EPOCH_NUM, \n        callbacks=[rlr, mcp]\n        )\nmodel_gelu.load_weights('weights.hdf5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Check result \ncompare result of relu and gelu. \non val_loss and val_acc, relu's score is better than gelu.   \nso, I use relu activation model."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot training & validation accuracy values\nplt.plot(history_relu.history['acc'])\nplt.plot(history_relu.history['val_acc'])\nplt.plot(history_gelu.history['acc'])\nplt.plot(history_gelu.history['val_acc'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train:relu', 'Test:relu', 'Train:gelu', 'Test:gelu'], loc='upper left')\nplt.show()\n\n# Plot training & validation loss values\nplt.plot(history_relu.history['loss'])\nplt.plot(history_relu.history['val_loss'])\nplt.plot(history_gelu.history['loss'])\nplt.plot(history_gelu.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train:relu', 'Test:relu', 'Train:gelu', 'Test:gelu'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('val_acc\\nrelu:{}\\ngelu:{}\\nval_loss\\nrelu:{}\\ngelu:{}'.format(max(history_relu.history['val_acc']), max(history_gelu.history['val_acc']), min(history_relu.history['val_loss']), min(history_gelu.history['val_loss'])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict by test data and create submission data.  \ncreate submission data by relu model. "},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_ret = None\nif min(history_relu.history['val_loss']) < min(history_gelu.history['val_loss']):\n    pred_ret = model_relu.predict_generator(test_datagen.flow(test_data, None, shuffle=False, batch_size=BATCH_SIZE))\nelse:\n    pred_ret = model_gelu.predict_generator(test_datagen.flow(test_data, None, shuffle=False, batch_size=BATCH_SIZE))\n    \npred_ids = np.argmax(pred_ret, axis=1)\npred_ids.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['label'] = pred_ids\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(path_or_buf='submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}