{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # data visualization\nimport seaborn as sns # data visualization\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\n\nimport tensorflow as tf\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, Dropout, Dense, Flatten, BatchNormalization, MaxPooling2D,LeakyReLU\nfrom keras.optimizers import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom keras.utils.np_utils import to_categorical\nfrom tensorflow.keras.regularizers import l2\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"model_name = 'k-mnist_trained_model.h5'\nnets = 5\nmodel = [0] *(nets)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"raw_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\nraw_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\n\nimport random\nrandom.seed(0)\nnp.random.seed(0)\nimport tensorflow as tf\ntf.compat.v1.set_random_seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def SelfKFold(raw_train):\n    '''\n    return n train and val dataset.\n    '''\n    from sklearn.model_selection import KFold\n    raw_train_val = raw_train\n    random_train_val = raw_train_val.sample(frac=1).reset_index(drop=True)#need fix np.random\n    kfold = KFold(n_splits=5,random_state=None)\n    train_val_data = raw_train_val.drop('label',axis = 1).values.astype(np.uint8)\n    train_val_label = raw_train_val['label'].values.astype(np.uint8)\n    print('data shape:', train_val_data.shape)\n    print('label shape:', train_val_label.shape)\n    train_total = []\n    train_label_total = []\n    val_total = []\n    val_label_total = []\n    for _,val_index in kfold.split(train_val_data,train_val_label):\n        print('val data index:',val_index[0],val_index[-1])\n        train_sub_0 = train_val_data[val_index[-1]+1:]\n        train_sub_1 = train_val_data[:val_index[0]]\n        val_sub = train_val_data[val_index[0]:val_index[-1]+1]\n        train_total.append(np.concatenate((train_sub_0,train_sub_1),axis=0))\n        val_total.append(val_sub)\n\n        train_label_sub_0 = train_val_label[val_index[-1]+1:]\n        train_label_sub_1 = train_val_label[:val_index[0]]\n        val_label_sub = train_val_label[val_index[0]:val_index[-1]+1]\n        train_label_total.append(np.concatenate((train_label_sub_0,train_label_sub_1),axis=0))\n        val_label_total.append(val_label_sub)\n    print('number of dataset:',len(val_total))\n    return train_total,train_label_total,val_total,val_label_total","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train_, y_train_,x_val_, y_val_ =  SelfKFold(raw_train)\nx_train_ = raw_train.drop('label',axis = 1).values\n#x_train_test = raw_test.drop('id',axis = 1).values\n#x_train_ = np.concatenate([x_train_,x_train_test],axis=0)\ny_train_ = raw_train['label'].values\n#y_train_ = np.concatenate([y_train_,results5000],axis=0)\nprint(x_train_.shape)\nprint(y_train_.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nfor j in range(nets):\n    model[j] = Sequential([\n        Conv2D(64, (3,3), padding='same', input_shape=(28, 28, 1)),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n        Conv2D(64,  (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n        Conv2D(64,  (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n\n        MaxPooling2D(2, 2),\n        Dropout(0.25),\n\n        Conv2D(128, (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n        Conv2D(128, (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n        Conv2D(128, (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n\n        MaxPooling2D(2,2),\n        Dropout(0.25),    \n\n        Conv2D(256, (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),\n        LeakyReLU(alpha=0.1),\n        Conv2D(256, (3,3), padding='same'),\n        BatchNormalization(momentum=0.9, epsilon=1e-5, gamma_initializer=\"uniform\"),##\n        LeakyReLU(alpha=0.1),\n\n        MaxPooling2D(2,2),\n        Dropout(0.25),\n\n\n        Flatten(),\n        Dense(256),\n        LeakyReLU(alpha=0.1),\n\n        BatchNormalization(),\n        Dense(10, activation='softmax')\n    ])\n    model[j].summary()\n#     optimizer = RMSprop(learning_rate=0.0025,###########\n#     rho=0.9,\n#     momentum=0.1,\n#     epsilon=1e-07,\n#     centered=True,\n#     name='RMSprop')\n    optimizer = RMSprop(lr=0.0025)\n    model[j].compile(loss= keras.losses.CategoricalCrossentropy(label_smoothing=0.1),\n              optimizer=optimizer,\n              metrics=['accuracy'])\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1024\nnum_classes = 10\nepochs = 56\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen_train = ImageDataGenerator(rotation_range = 10,\n                                   width_shift_range = 0.25,\n                                   height_shift_range = 0.25,\n                                   shear_range = 10,\n                                   zoom_range = 0.40,\n                                   horizontal_flip = False)\n\ndatagen_val = ImageDataGenerator() \ndef schedule(epoch):\n    if epoch >= 52:\n        lr = 1e-5\n    elif epoch >= 48:\n        lr = 0.0025 * 0.25 * 0.25 * 0.25\n    elif epoch >= 43:\n        lr = 0.0025 * 0.25 * 0.25\n    elif epoch >= 35:\n        lr = 0.0025 * 0.25\n    else:\n        lr = 0.0025\n    return lr\n\nlearning_rate_reduction = tf.keras.callbacks.LearningRateScheduler(schedule, verbose=1)\n\n# learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau( \n#     monitor='loss',    # Quantity to be monitored.\n#     factor=0.25,       # Factor by which the learning rate will be reduced. new_lr = lr * factor\n#     patience=2,        # The number of epochs with no improvement after which learning rate will be reduced.\n#     verbose=1,         # 0: quiet - 1: update messages.\n#     mode=\"auto\",       # {auto, min, max}. In min mode, lr will be reduced when the quantity monitored has stopped decreasing; \n#                        # in the max mode it will be reduced when the quantity monitored has stopped increasing; \n#                        # in auto mode, the direction is automatically inferred from the name of the monitored quantity.\n#     min_delta=0.0001,  # threshold for measuring the new optimum, to only focus on significant changes.\n#     cooldown=0,        # number of epochs to wait before resuming normal operation after learning rate (lr) has been reduced.\n#     min_lr=0.00001     # lower bound on the learning rate.\n#     )\n\n#es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=300, restore_best_weights=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for j in range(nets):\n#     loadmodelname = \"../input/kernel41bf03341f/weights_N\" + str(j)\n#     model[j].load_weights(loadmodelname)\n#     savemodelname = \"weights_N\" + str(j)\n#     model[j].save_weights(savemodelname)\n#     print(\"Saving\", loadmodelname, \"back to\", savemodelname)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint\nnets2train = 5\nhistory = [0] * nets2train\nfor j in range(nets2train):\n    x_train = x_train_.reshape(-1, 28, 28,1).astype('float32') / 255\n   # print(x_train.shape)\n    #x_val = x_val_[j].reshape(-1, 28, 28,1).astype('float32') / 255\n    x_val = x_train[:1024,:,:,:]\n    y_train = to_categorical(y_train_)\n    #y_val = to_categorical(y_val_[j])\n    y_val = y_train[:1024,:]\n    #print(y_train.shape)\n    modelfilename = \"weights_N_\" + str(j)\n    #modelfilename_val = \"weights_N_val_\" + str(j)\n    checkpoint = ModelCheckpoint(modelfilename, monitor='acc', verbose=0, save_best_only=False, save_weights_only=False, period=epochs)\n    #checkpoint_val = ModelCheckpoint(modelfilename_val, monitor='val_acc', verbose=0, save_best_only=True, save_weights_only=False, period=epochs)\n    history[j] = model[j].fit_generator(datagen_train.flow(x_train, y_train, batch_size=batch_size),\n                              steps_per_epoch=len(x_train)//batch_size,\n                              epochs=epochs,\n                              validation_data=(x_val, y_val),\n                              validation_steps=1,\n                              callbacks=[learning_rate_reduction,checkpoint],\n                              verbose=2)\n    #model[j].save_weights(modelfilename)\n    print(\"CNN\", j,\": Training done.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test5000 = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\nX_test5000 = test5000.drop(labels = [\"id\"],axis = 1)\nX_test5000 = X_test5000 / 255.0\nX_test5000 = X_test5000.values.reshape(-1,28,28,1)\nTTA = 6\nnets4predict = 5\ndatagen_test = [0]*TTA\nresults5000 = np.zeros( (X_test5000.shape[0], 10) ) \nallthree = 0\npreds_tta = []\nfor each_test in range(TTA):\n    if allthree :\n        if each_test == 0:\n            datagen_test[each_test] = ImageDataGenerator(#rotation_range = 10,\n                                           #width_shift_range = 0.25,\n                                           #height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           zoom_range = 0.4,\n                                           horizontal_flip = False,\n               \n                                           )\n        elif each_test == 1:\n            datagen_test[each_test] = ImageDataGenerator(#rotation_range = 10,\n                                           width_shift_range = 0.25,\n                                           #height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           #zoom_range = 0.4,\n                                           horizontal_flip = False,\n             \n                                           )\n        elif each_test == 2:\n            datagen_test[each_test] = ImageDataGenerator(#rotation_range = 10,\n                                           #width_shift_range = 0.25,\n                                           height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           #zoom_range = 0.4,\n                                           horizontal_flip = False,\n               \n                                           )\n        elif each_test == 3:\n            datagen_test[each_test] = ImageDataGenerator(rotation_range = 10,\n                                           #width_shift_range = 0.25,\n                                           #height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           #zoom_range = 0.4,\n                                           horizontal_flip = False,\n                                                       \n                                           )\n        elif each_test == 4:\n            datagen_test[each_test] = ImageDataGenerator(#rotation_range = 10,\n                                           #width_shift_range = 0.25,\n                                           #height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           zoom_range = 0.2,\n                                           horizontal_flip = False,\n               \n                                           )\n        elif each_test == 5:\n            datagen_test[each_test] = ImageDataGenerator(#rotation_range = 10,\n                                           #width_shift_range = 0.25,\n                                           #height_shift_range = 0.25,\n                                           #shear_range = 10,\n                                           #zoom_range = 0.4,\n                                           horizontal_flip = False,\n                                           \n                                           )\n        test_generator = datagen_test[each_test].flow(\n                X_test5000,\n                shuffle = False,\n                #class_mode='categorical',\n                batch_size=500)\n        \n        for j in range(nets4predict):\n            print(\"CNN\",j)\n            loadmodelname = \"weights_N\" + str(j)\n            model[j].load_weights(loadmodelname)\n            test_generator.reset()\n            preds = model[j].predict_generator(\n                generator=test_generator,\n                steps =int(X_test5000.shape[0]/500),\n            )\n            print(preds.shape)\n            preds_tta.append(preds)\n            \n\n\n#             for x_batch in datagen_test[each_test].flow(X_test5000,batch_size=5000,shuffle=False):\n#                 print('here')\n#                 ttt = x_batch\n#                 break\n#             results5000 = results5000 + model[j].predict(ttt)\nresults5000 = np.mean(preds_tta,axis = 0)\nresults5000 = np.argmax(results5000,axis = 1)\n\nsubmission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\nsubmission['label'] = results5000\nsubmission.to_csv(\"submission.csv\",index=False)\n\nprint(\"DONE.\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}