{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Input, Dense, LeakyReLU, Flatten, Dropout, BatchNormalization,Add,GlobalAveragePooling2D,Softmax\nfrom tensorflow.keras.callbacks import LearningRateScheduler,ModelCheckpoint,EarlyStopping\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport math\nfrom sklearn.model_selection import train_test_split\nimport random","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_datas = pd.read_csv(\"../input/Kannada-MNIST/train.csv\")\nval_datas = pd.read_csv(\"../input/Kannada-MNIST/Dig-MNIST.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_datas.shape)\nprint(val_datas.shape)\n#train_datas = train_datas / 255.0\n#val_datas = val_datas / 255.0\n\n#concat train and val\ndatas = pd.concat([train_datas,val_datas],axis=0)\nprint(datas.shape)\ndatas_X = np.array(datas.drop(\"label\",axis=1),dtype=np.float32)\ndatas_Y = np.array(datas[[\"label\"]],dtype=np.int32)\ntrain_X,val_X,train_Y,val_Y = train_test_split(datas_X,datas_Y,test_size=0.1,shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = np.reshape(train_X,(-1,28,28,1))\nval_X = np.reshape(val_X,(-1,28,28,1))\n\nprint(train_X.shape,train_X.dtype)\nprint(train_Y.shape,train_Y.dtype)\nprint(val_X.shape,val_X.dtype)\nprint(val_Y.shape,val_Y.dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# models_blocks = [\n#     [[64,128,2],[128,256,4],[256,10,2]]\n# ]\n# CLASS_NUMS = 10\n# #####\n# tf.reset_default_graph()\n# nets = len(models_blocks)\n# model = [0] * nets\n# for j,blocks in enumerate(models_blocks):\n#     input_layer = Input(shape=(28,28,1))\n#     net = Conv2D(64, (3,3), padding='same')(input_layer)\n#     net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n#     net = LeakyReLU(alpha=0.1)(net)\n#     net = Conv2D(64, (3,3), padding='same')(input_layer)\n#     net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n#     net = LeakyReLU(alpha=0.1)(net)\n    \n#     for block in blocks:\n#         net = Conv2D(block[1], (3,3) ,padding='same')(net)\n#         net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n#         if len(block) <= 3 or block[3]==True:\n#             net = MaxPooling2D(2,2)(net)\n#         net = LeakyReLU(alpha=0.1)(net)\n#         shortcut = net\n#         for count in range(block[2]):\n#             net = Conv2D(block[0], (3,3), padding='same')(net)\n#             net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n#             net = LeakyReLU(alpha=0.1)(net)\n#             net = Conv2D(block[1], (3,3), padding='same')(net)\n#             net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n#             net = LeakyReLU(alpha=0.1)(net)\n#         net = Add()([net,shortcut])\n# #         if len(block) > 3:\n# #             net = Dropout(block[3])(net)\n# #     net = Conv2D(CLASS_NUMS, (3,3), padding='same')(net)\n# #     net = BatchNormalization(momentum=0.95, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n# #     net = LeakyReLU(alpha=0.1)(net)\n#     logits = GlobalAveragePooling2D()(net)\n#     output = Softmax()(logits)\n    \n#     model[j] = Model(inputs=input_layer,outputs=output)\n#     model[j].compile(optimizer='adam', loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])\n# print(model[0].summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.reset_default_graph()\nnets = 1\nmodel = [0]*nets\nfor j in range(nets):\n    input_layer = Input(shape=(28,28,1))\n    net = Conv2D(64, (3,3), padding='same')(input_layer)\n    net = BatchNormalization(momentum=0.5, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n    net = LeakyReLU(alpha=0.1)(net)\n    net = Conv2D(64,  (3,3), padding='same')(net)\n    net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n\n    net = LeakyReLU(alpha=0.1)(net)\n\n    net = MaxPooling2D(2, 2)(net)\n\n    net = Conv2D(128, (3,3), padding='same')(net)\n    net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n    net = LeakyReLU(alpha=0.1)(net)\n    net = Conv2D(128, (3,3), padding='same')(net)\n    net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n\n    net = LeakyReLU(alpha=0.1)(net)\n\n    net = MaxPooling2D(2,2)(net)\n    net = Dropout(0.2)(net) \n\n    net = Conv2D(256, (3,3), padding='same')(net)\n    net = BatchNormalization(momentum=0.2, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n    net = LeakyReLU(alpha=0.1)(net)\n\n    net = Conv2D(256, (3,3), padding='same')(net)\n    net = BatchNormalization(momentum=0.1, epsilon=1e-5, gamma_initializer=\"uniform\")(net)\n    net = LeakyReLU(alpha=0.1)(net)\n\n\n    net = MaxPooling2D(2,2)(net)\n    net = Dropout(0.2)(net)\n    \n    net = GlobalAveragePooling2D()(net)\n    \n    net = BatchNormalization()(net)\n    output = Dense(10, activation='softmax')(net)\n    model[j] = Model(inputs=input_layer,outputs=output)\n    model[j].compile(optimizer='Adam', loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])\nprint(model[0].summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range = 20,\n    width_shift_range = 0.25,\n    height_shift_range = 0.25,\n    shear_range = 0.1,\n    zoom_range = 0.25,\n    horizontal_flip = False,\n    rescale=1/255.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#val_datagen = ImageDataGenerator(rescale=1/255.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"annealer = LearningRateScheduler(lambda x: 0.002 * 0.99 ** x)\nmodel_checks = []\nfor j in range(nets):\n    model_checks.append(ModelCheckpoint(\"model-{}.ckpt\".format(j), monitor='val_acc', verbose=1, save_best_only=True,))\nearly_stop = EarlyStopping(monitor='val_acc', verbose=2, patience=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = [0] * nets\nepochs = 150\nbatch_size = 1024\nfor j in range(nets):\n    history[j] = model[j].fit_generator(datagen.flow(train_X,train_Y, batch_size=batch_size),\n                                        epochs = epochs, \n                                        steps_per_epoch = math.ceil(train_X.shape[0]/batch_size)*2,\n                                        validation_data = (val_X/255.0,val_Y),\n                                        callbacks=[annealer,model_checks[j],early_stop],\n                                        verbose=2)\n    print(\"Train: Model{:d}: Epochs={:d},LearningRate={:.12f} ,Train accuracy={:.5f}, Validation accuracy={:.5f}\".format(\n        j+1,epochs,tf.keras.backend.get_value(model[j].optimizer.lr),max(history[j].history['acc']),max(history[j].history['val_acc'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv = pd.read_csv(\"../input/Kannada-MNIST/test.csv\")\ndata_test = np.array(test_csv.drop(\"id\",axis=1),dtype=np.float32)\ndata_test = np.reshape(data_test,(-1,28,28,1))\nX_test = data_test / 255.0\n#X_test = data_test\nprint(X_test.shape,X_test.dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for j in range(nets):\n#     model[j].load_weights(\"model-{}.ckpt\".format(j))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ENSEMBLE\n# results = np.zeros( (X_test.shape[0],10))\n# for j in range(nets):\n#     results = results + model[j].predict(X_test)\n# results = np.argmax(results,axis = 1)\n# results = results[:,None]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_test1,x_nouse,y_test1,y_nouse = train_test_split(X_test,results,test_size=0.05,shuffle=True)\n# new_X = np.concatenate([train_X,val_X,x_test1],axis=0)\n# new_Y = np.concatenate([train_Y,val_Y,y_test1],axis=0)\n# print(new_X.shape)\n# print(new_Y.shape)\n# new_train_X,new_val_X,new_train_Y,new_val_Y = train_test_split(new_X,new_Y,test_size=0.1,shuffle=True)\n# print(new_train_X.shape)\n# print(new_train_Y.shape)\n# print(new_val_X.shape)\n# print(new_val_Y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history = [0] * nets\n# epochs = 30\n# batch_size = 128\n# for j in range(nets):\n#     history[j] = model[j].fit_generator(datagen.flow(new_train_X,new_train_Y, batch_size=batch_size),\n#         epochs = epochs, steps_per_epoch = math.ceil(new_train_X.shape[0]*1.0/batch_size),  \n#         validation_data = (new_val_X,new_val_Y), callbacks=[model_checks[j],annealer],verbose=1)\n#     print(\"Train: Model{:d}: Epochs={:d},LearningRate={:.12f} ,Train accuracy={:.5f}, Validation accuracy={:.5f}\".format(\n#         j+1,epochs,tf.keras.backend.get_value(model[j].optimizer.lr),max(history[j].history['acc']),max(history[j].history['val_acc'])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for j in range(nets):\n    model[j].load_weights(\"model-{}.ckpt\".format(j))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ENSEMBLE\nresults = np.zeros( (X_test.shape[0],10))\nfor j in range(nets):\n    results = results + model[j].predict(X_test)\nresults = np.argmax(results,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(\"../input/Kannada-MNIST/sample_submission.csv\")\nsubmission['label'] = results\nsubmission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}