{"cells":[{"metadata":{},"cell_type":"markdown","source":"![]()"},{"metadata":{"_uuid":"b1ca1df7f5915973755864390b0003fa660cf60d"},"cell_type":"markdown","source":"# MNIST Classification"},{"metadata":{},"cell_type":"markdown","source":"![](https://camo.githubusercontent.com/d440ac2eee1cb3ea33340a2c5f6f15a0878e9275/687474703a2f2f692e7974696d672e636f6d2f76692f3051493378675875422d512f687164656661756c742e6a7067)"},{"metadata":{"_uuid":"11aa55f3db0dec51c09334c64fae7f56b1a2cde1"},"cell_type":"markdown","source":"# 1. Import "},{"metadata":{"_uuid":"ae198668fcf74479c92b2dfbf2dc3f291b599d0e","trusted":true},"cell_type":"code","source":"# System\nimport sys\nimport os\nimport argparse\nimport itertools\n\n# Time\nimport time\nimport datetime\n\n# Numerical Data\nimport random\nimport numpy as np \nimport pandas as pd\n\n# Tools\nimport shutil\nfrom glob import glob\nfrom tqdm import tqdm\nimport gc\n\n# NLP\nimport re\n\n# Preprocessing\nfrom sklearn import preprocessing\nfrom sklearn.utils import class_weight as cw\nfrom sklearn.utils import shuffle\n\n# Model Selection\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import cross_val_score\n\n# Machine Learning Models\nfrom sklearn import svm\nfrom sklearn.svm import LinearSVC, SVC\n\n# Evaluation Metrics\nfrom sklearn import metrics \nfrom sklearn.metrics import f1_score, accuracy_score, precision_score, recall_score, confusion_matrix, classification_report, roc_auc_score\n\n\n# Deep Learning - Keras -  Preprocessing\nfrom keras.preprocessing.image import ImageDataGenerator\n\n# Deep Learning - Keras - Model\nimport keras\nfrom keras import models\nfrom keras.models import Model\nfrom keras.models import load_model\nfrom keras.models import Sequential\n\n# Deep Learning - Keras - Layers\nfrom keras.layers import Convolution1D, concatenate, SpatialDropout1D, GlobalMaxPool1D, GlobalAvgPool1D, Embedding, \\\n    Conv2D, SeparableConv1D, Add, BatchNormalization, Activation, GlobalAveragePooling2D, LeakyReLU, Flatten\nfrom keras.layers import Dense, Input, Dropout, MaxPool2D, MaxPooling2D, Concatenate, GlobalMaxPooling2D, GlobalAveragePooling2D, \\\n    Lambda, Multiply, LSTM, Bidirectional, PReLU, MaxPooling1D\nfrom keras.layers.pooling import _GlobalPooling1D\n\nfrom keras.regularizers import l2\n\n# Deep Learning - Keras - Pretrained Models\nfrom keras.applications.xception import Xception\nfrom keras.applications.resnet50 import ResNet50\nfrom keras.applications.inception_v3 import InceptionV3\nfrom keras.applications.inception_resnet_v2 import InceptionResNetV2\nfrom keras.applications.densenet import DenseNet201\nfrom keras.applications.nasnet import NASNetMobile, NASNetLarge\n\nfrom keras.applications.nasnet import preprocess_input\n\n# Deep Learning - Keras - Model Parameters and Evaluation Metrics\nfrom keras import optimizers\nfrom keras.optimizers import Adam, SGD , RMSprop\nfrom keras.losses import mae, sparse_categorical_crossentropy, binary_crossentropy\n\n# Deep Learning - Keras - Visualisation\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau, LearningRateScheduler\n# from keras.wrappers.scikit_learn import KerasClassifier\nfrom keras import backend as K\n\n# Deep Learning - TensorFlow\nimport tensorflow as tf\n\n# Graph/ Visualization\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nimport matplotlib.image as mpimg\nimport seaborn as sns\nfrom mlxtend.plotting import plot_confusion_matrix\n\n# Image\nimport cv2\nfrom PIL import Image\nfrom IPython.display import display\n\n# np.random.seed(42)\n\n%matplotlib inline\n\n# Input data\nprint(os.listdir(\"../input/\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ec897e74117506f17611b58af485a2e013e848"},"cell_type":"markdown","source":"# 2. Functions"},{"metadata":{"_uuid":"cbe4bc8e76957ae5b139e87a19e75e51be7af414","trusted":true},"cell_type":"code","source":"def date_time(x):\n    if x==1:\n        return 'Timestamp: {:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())\n    if x==2:    \n        return 'Timestamp: {:%Y-%b-%d %H:%M:%S}'.format(datetime.datetime.now())\n    if x==3:  \n        return 'Date now: %s' % datetime.datetime.now()\n    if x==4:  \n        return 'Date today: %s' % datetime.date.today()  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cc1f975dabb307ccf1c219b5021dfc4a97a7a117"},"cell_type":"markdown","source":"# 3. Input Configuration"},{"metadata":{"_uuid":"a8a36cda21a37ddf8002c994b682d91a81c59b56","trusted":true},"cell_type":"code","source":"input_directory = r\"../input/digit-recognizer/\"\noutput_directory = r\"../output/\"\n\ntraining_dir = input_directory + \"train_images\"\ntesting_dir = input_directory + r\"test_images\"\n\nif not os.path.exists(output_directory):\n    os.mkdir(output_directory)\n    \nfigure_directory = \"../output/figures\"\nif not os.path.exists(figure_directory):\n    os.mkdir(figure_directory)\n\n# model_input_directory = \"../input/models/\"\n# if not os.path.exists(model_input_directory):\n#     os.mkdir(model_input_directory)\n\nmodel_output_directory = \"../output/models/\"\nif not os.path.exists(model_output_directory):\n    os.mkdir(model_output_directory)\n\n\n    \nfile_name_pred_batch = figure_directory+r\"/result\"\nfile_name_pred_sample = figure_directory+r\"/sample\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2bd626b286efbc7ca36291ad5b856c25d8c2da52","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(input_directory + \"train.csv\")\ntrain_df.rename(index=str, columns={\"label\": \"target\"}, inplace=True)\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7abcb11ec076b4c916a7ea219fe57e9f03114a2a","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(input_directory + \"test.csv\")\ntest_df.rename(index=str, columns={\"label\": \"target\"}, inplace=True)\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9e02a5a1b4538b67bf351ec59a9cfd7f832fb0e3"},"cell_type":"markdown","source":"# 4. Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"ticksize = 18\ntitlesize = ticksize + 8\nlabelsize = ticksize + 5\n\nfigsize = (18, 5)\nparams = {'figure.figsize' : figsize,\n          'axes.labelsize' : labelsize,\n          'axes.titlesize' : titlesize,\n          'xtick.labelsize': ticksize,\n          'ytick.labelsize': ticksize}\n\nplt.rcParams.update(params)\n\ncol = \"target\"\nxlabel = \"Label\"\nylabel = \"Count\"\n\nsns.countplot(x=train_df[col])\nplt.title(\"Label Count\")\nplt.xlabel(xlabel)\nplt.ylabel(ylabel)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08472c5647d039fe88134dd5bbf51b1bacafe700"},"cell_type":"markdown","source":"# 5. Preprocess"},{"metadata":{"_uuid":"5f0a5bf3d06e33bec36a8b9355483d3432d5c63f","trusted":true},"cell_type":"code","source":"def get_data(train_X=None, train_Y=None, test_X=None, batch_size=32):\n    print(\"Preprocessing and Generating Data Batches.......\\n\")\n    \n    rescale = 1.0/255\n\n    train_batch_size = batch_size\n    validation_batch_size = batch_size*5\n    test_batch_size = batch_size*5\n    \n    train_shuffle = True\n    val_shuffle = True\n    test_shuffle = False\n    \n    train_datagen = ImageDataGenerator(\n        horizontal_flip=False,\n        vertical_flip=False,\n        rotation_range=10,\n#         shear_range=15,\n        zoom_range = 0.1, \n        width_shift_range=0.1,  \n        height_shift_range=0.1,\n        rescale=rescale,\n        validation_split=0.25)\n    \n    train_generator = train_datagen.flow(\n        x=train_X, \n        y=train_Y, \n        batch_size=batch_size,\n        shuffle=True, \n        sample_weight=None, \n        seed=42, \n        save_to_dir=None, \n        save_prefix='', \n        save_format='png', \n        subset='training')\n    \n    \n    validation_generator = train_datagen.flow(\n        x=train_X, \n        y=train_Y, \n        batch_size=validation_batch_size,\n        shuffle=False, \n        sample_weight=None, \n        seed=42, \n        save_to_dir=None, \n        save_prefix='', \n        save_format='png', \n        subset='validation')\n    \n    test_datagen = ImageDataGenerator(rescale=rescale)\n    \n    test_generator = test_datagen.flow(\n        x=test_X, \n        y=None,  \n        batch_size=test_batch_size,\n        shuffle=False, \n        sample_weight=None, \n        seed=42, \n        save_to_dir=None, \n        save_prefix='', \n        save_format='png')\n    \n    class_weights = get_weight(np.argmax(train_Y, axis=1))\n    \n    steps_per_epoch = len(train_generator)\n    validation_steps = len(validation_generator)\n    \n    print(\"\\nPreprocessing and Data Batch Generation Completed.\\n\")\n    \n    \n    return train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps\n            \n# Calculate Class Weights\ndef get_weight(y):\n    class_weight_current =  cw.compute_class_weight('balanced', np.unique(y), y)\n    return class_weight_current","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e02af0cac288607e1ae124f8242db9b1c05688c8"},"cell_type":"markdown","source":"# 5. Model Function"},{"metadata":{"_uuid":"868b5798867b18a80f8d5b12957cd7965a0b65ae","trusted":true},"cell_type":"code","source":"def get_model(model_name, input_shape=(96, 96, 3), num_class=2, weights='imagenet', dense_units=1024, internet=False):\n    inputs = Input(input_shape)\n    \n    if model_name == \"Xception\":\n        base_model = Xception(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet50\":\n        base_model = ResNet50(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet101\":\n        base_model = keras.applications.resnet.ResNet101(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet152\":\n        base_model = keras.applications.resnet.ResNet152(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet50V2\":\n        base_model = resnet_v2.ResNet50V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet101V2\":\n        base_model = resnet_v2.ResNet101V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNet152V2\":\n        base_model = resnet_v2.ResNet152V2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNeXt50\":\n        base_model = resnext.ResNeXt50(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"ResNeXt101\":\n        base_model = resnext.ResNeXt101(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"InceptionV3\":\n        base_model = InceptionV3(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"InceptionResNetV2\":\n        base_model = InceptionResNetV2(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"DenseNet201\":\n        base_model = DenseNet201(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"NASNetMobile\":\n        base_model = NASNetMobile(include_top=False, weights=weights, input_shape=input_shape)\n    elif model_name == \"NASNetLarge\":\n        base_model = NASNetLarge(include_top=False, weights=weights, input_shape=input_shape)\n        \n        \n#     x = base_model(inputs)\n#     x = Dropout(0.5)(x)\n    \n#     out1 = GlobalMaxPooling2D()(x)\n#     out2 = GlobalAveragePooling2D()(x)\n#     out3 = Flatten()(x)\n    \n#     out = Concatenate(axis=-1)([out1, out2, out3])\n    \n#     out = Dropout(0.6)(out)\n#     out = BatchNormalization()(out)\n#     out = Dropout(0.5)(out)\n    \n#     if num_class>1:\n#         out = Dense(num_class, activation=\"softmax\", name=\"3_\")(out)\n#     else:\n#         out = Dense(1, activation=\"sigmoid\", name=\"3_\")(out)\n        \n#     model = Model(inputs, out)\n#     model = Model(inputs=base_model.input, outputs=outputs)\n\n    \n    x = base_model.output\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    x = GlobalAveragePooling2D()(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    \n    x = Dense(dense_units)(x)\n    x = BatchNormalization()(x)\n    x = Activation(activation='relu')(x)\n    x = Dropout(0.5)(x)\n    \n    if num_class>1:\n        outputs = Dense(num_class, activation=\"softmax\")(x)\n    else:\n        outputs = Dense(1, activation=\"sigmoid\")(x)\n\n    model = Model(inputs=base_model.input, outputs=outputs)\n    \n    model.summary()\n    \n    \n    return model\n\n\n# def get_conv_model(num_class=2, input_shape=None, dense_size=256):\n#     model = Sequential()\n\n#     model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', input_shape = input_shape))\n#     model.add(BatchNormalization())\n#     model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n    \n#     model.add(MaxPool2D(pool_size=(2,2)))\n#     model.add(Dropout(0.25))\n\n\n#     model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n#     model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n    \n#     model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n#     model.add(Dropout(0.25))\n\n    \n#     model.add(Flatten())\n#     model.add(Dense(256, activation = \"relu\"))\n#     model.add(Dropout(0.5))\n\n    \n# #     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4), input_shape = input_shape))\n# #     model.add(BatchNormalization())\n# #     model.add(Activation('relu'))\n    \n# #     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n# #     model.add(BatchNormalization())\n# #     model.add(Activation('relu'))\n    \n# #     model.add(MaxPool2D())\n# #     model.add(Dropout(0.5))\n\n    \n# #     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n# #     model.add(BatchNormalization())\n# #     model.add(Activation('relu'))\n    \n# #     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n# #     model.add(BatchNormalization())\n# #     model.add(Activation('relu'))\n    \n# #     model.add(MaxPool2D())\n# #     model.add(Dropout(0.5))\n    \n    \n# #     model.add(GlobalAveragePooling2D())\n    \n    \n#     if num_class>1:\n#         model.add(Dense(num_class, activation='softmax'))\n#     else:\n#         model.add(Dense(num_class, activation='sigmoid'))\n    \n#     print(model.summary())\n\n#     return model\n\n\ndef get_conv_model(num_class=2, input_shape=None, dense_units=256):\n    model = Sequential()\n\n    model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', input_shape = input_shape))\n    model.add(BatchNormalization())\n    model.add(Conv2D(filters = 32, kernel_size = (5,5), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPool2D(pool_size=(2,2)))\n    model.add(Dropout(0.25))\n\n\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    model.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same', activation ='relu', kernel_regularizer=l2(1e-4)))\n    model.add(BatchNormalization())\n    \n    model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n    model.add(Dropout(0.25))\n\n    \n    model.add(Flatten())\n    model.add(Dense(dense_units, activation = \"relu\"))\n    model.add(Dropout(0.5))\n\n    \n#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4), input_shape = input_shape))\n#     model.add(BatchNormalization())\n#     model.add(Activation('relu'))\n    \n#     model.add(Conv2D(32, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n#     model.add(Activation('relu'))\n    \n#     model.add(MaxPool2D())\n#     model.add(Dropout(0.5))\n\n    \n#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n#     model.add(Activation('relu'))\n    \n#     model.add(Conv2D(64, (3, 3), padding='same', use_bias=False, kernel_regularizer=l2(1e-4)))\n#     model.add(BatchNormalization())\n#     model.add(Activation('relu'))\n    \n#     model.add(MaxPool2D())\n#     model.add(Dropout(0.5))\n    \n    \n#     model.add(GlobalAveragePooling2D())\n    \n    \n    if num_class>1:\n        model.add(Dense(num_class, activation='softmax'))\n    else:\n        model.add(Dense(num_class, activation='sigmoid'))\n    \n    print(model.summary())\n\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualization Function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_performance(history=None, figure_directory=None):\n    xlabel = 'Epoch'\n    legends = ['Training', 'Validation']\n\n    ylim_pad = [0.005, 0.005]\n    ylim_pad = [0, 0]\n\n\n    plt.figure(figsize=(20, 5))\n\n    # Plot training & validation Accuracy values\n\n    y1 = history.history['accuracy']\n    y2 = history.history['val_accuracy']\n\n    min_y = min(min(y1), min(y2))-ylim_pad[0]\n    max_y = max(max(y1), max(y2))+ylim_pad[0]\n    \n#     min_y = .96\n#     max_y = 1\n\n\n    plt.subplot(121)\n\n    plt.plot(y1)\n    plt.plot(y2)\n\n    plt.title('Model Accuracy\\n'+date_time(1), fontsize=17)\n    plt.xlabel(xlabel, fontsize=15)\n    plt.ylabel('Accuracy', fontsize=15)\n    plt.ylim(min_y, max_y)\n    plt.legend(legends, loc='upper left')\n    plt.grid()\n\n\n    # Plot training & validation loss values\n\n    y1 = history.history['loss']\n    y2 = history.history['val_loss']\n\n    min_y = min(min(y1), min(y2))-ylim_pad[1]\n    max_y = max(max(y1), max(y2))+ylim_pad[1]\n\n#     min_y = .1\n#     max_y = 0\n\n    plt.subplot(122)\n\n    plt.plot(y1)\n    plt.plot(y2)\n\n    plt.title('Model Loss\\n'+date_time(1), fontsize=17)\n    plt.xlabel(xlabel, fontsize=15)\n    plt.ylabel('Loss', fontsize=15)\n    plt.ylim(min_y, max_y)\n    plt.legend(legends, loc='upper left')\n    plt.grid()\n    if figure_directory:\n        plt.savefig(figure_directory+\"/history\")\n\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e56fcfe7a60a2759f1952f30f7fb05726b022c5"},"cell_type":"markdown","source":"# 6. Output Configuration"},{"metadata":{"_uuid":"644c92b9756bb9fd1d980302a315d38298aa8472","trusted":true},"cell_type":"code","source":"main_model_dir = output_directory + r\"models_output/\"\nmain_log_dir = output_directory + r\"logs/\"\n\ntry:\n    os.mkdir(main_model_dir)\nexcept:\n    print(\"Could not create main model directory\")\n    \ntry:\n    os.mkdir(main_log_dir)\nexcept:\n    print(\"Could not create main log directory\")\n\n\n\nmodel_dir = main_model_dir + time.strftime('%Y-%m-%d %H-%M-%S') + \"/\"\nlog_dir = main_log_dir + time.strftime('%Y-%m-%d %H-%M-%S')\n\n\ntry:\n    os.mkdir(model_dir)\nexcept:\n    print(\"Could not create model directory\")\n    \ntry:\n    os.mkdir(log_dir)\nexcept:\n    print(\"Could not create log directory\")\n    \nmodel_file = model_dir + \"{epoch:02d}-val_acc-{val_acc:.2f}-val_loss-{val_loss:.2f}.hdf5\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5133db17f6ff324823b131eacc3fb471943adc1"},"cell_type":"markdown","source":"## 6.2 Call Back Configuration"},{"metadata":{"_uuid":"104999913bc3bd3f1181b365fff5cc23f9a56ce7","trusted":true},"cell_type":"code","source":"print(\"Settting Callbacks\")\n\ndef step_decay(epoch, lr):\n    # initial_lrate = 1.0 # no longer needed\n    lrate = lr\n    if epoch==2:\n        lrate = 0.0001  \n#     lrate = lr * math.pow(drop, math.floor((1+epoch)/epochs_drop))\n    return lrate\n\n\ncheckpoint = ModelCheckpoint(\n    model_file, \n    monitor='val_acc', \n    save_best_only=True)\n\nearly_stopping = EarlyStopping(\n    monitor='val_loss',\n    patience=10,\n    verbose=1,\n    restore_best_weights=True)\n\n\nreduce_lr = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.6,\n    patience=2,\n    min_lr=0.0000001,\n    verbose=1)\n\nlearning_rate_scheduler = LearningRateScheduler(step_decay, verbose=1)\n# f1_metrics = Metrics()\n\n\ncallbacks = [reduce_lr, early_stopping]\n# callbacks = [checkpoint, reduce_lr, early_stopping]\n# callbacks = [reduce_lr, early_stopping, f1_metrics]\n\nprint(\"Set Callbacks at \", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6fbe80b2f63f0b9972c1546b088b667331108aa5"},"cell_type":"markdown","source":"# 7. Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Getting Base Model\", date_time(1))\n\n# model_name=\"InceptionV3\"\n# model_name=\"NASNetMobile\"\n\ndim = 28\n\ninput_shape = (dim, dim, 1)\n\n\nnum_class = len(set(train_df[\"target\"].values))\n\nweights = 'imagenet'\ndense_units = 256\n\n# internet = True\n\n# model = get_model(model_name=model_name, \n#                   input_shape=input_shape, \n#                   num_class=num_class, \n#                   weights=weights, \n#                   dense_units=dense_units, \n#                   internet=internet)\n\nmodel = get_conv_model(num_class=num_class, input_shape=input_shape, dense_units=dense_units)\nprint(\"Loaded Base Model\", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77a593fffaf1b82c2b82bbfd8d5335bc7995a6bf","trusted":true},"cell_type":"code","source":"loss = 'categorical_crossentropy'\n# loss = 'binary_crossentropy'\nmetrics = ['accuracy']\n# metrics = [auroc]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c975187a17a5e9d07ff15484cac2824fcab1fd53"},"cell_type":"markdown","source":"# 8. Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_X = train_df.drop(columns=[\"target\"]).values\n# train_Y = train_df[\"target\"].values\n\n\n# clf = svm.SVC()\n\n# cross_val_score(clf, train_X, train_Y, cv=10, n_jobs=-1, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X = train_df.drop(columns=[\"target\"]).values\ntrain_X = train_X.reshape(train_X.shape[0], dim, dim,1)\n\ntrain_Y = train_df[\"target\"].values\ntrain_Y = keras.utils.to_categorical(train_Y, 10) \n\ntest_X = test_df.values\ntest_X = test_X.reshape(test_X.shape[0], dim, dim,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 128\n\n# class_mode = \"categorical\"\n# class_mode = \"binary\"\n\n# target_size = (dim, dim)\n\ntrain_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(train_X=train_X, train_Y=train_Y, test_X=test_X, batch_size=batch_size)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd11c2eb7719f3f866c9e9ee201ca67e556fc73b"},"cell_type":"markdown","source":"# 9. Training"},{"metadata":{"_uuid":"5ac153e86d2338beab9cb0a3693f9c61fe7d3f04","trusted":true},"cell_type":"code","source":"print(\"Starting Trainning ...\\n\")\n\nstart_time = time.time()\nprint(date_time(1))\n\n# batch_size = 32\n# train_generator, validation_generator, test_generator, class_weights, steps_per_epoch, validation_steps = get_data(batch_size=batch_size)\n\nprint(\"\\n\\nCompliling Model ...\\n\")\nlearning_rate = 0.001\noptimizer = Adam(learning_rate)\n# optimizer = Adam()\n\nmodel.compile(optimizer=optimizer, loss=loss, metrics=metrics)\n\nsteps_per_epoch = len(train_generator)\nvalidation_steps = len(validation_generator)\n\nverbose = 1\nepochs = 100\n\nprint(\"Trainning Model ...\\n\")\nhistory = model.fit_generator(\n    train_generator,\n    steps_per_epoch=steps_per_epoch,\n    epochs=epochs,\n    verbose=verbose,\n    callbacks=callbacks,\n    validation_data=validation_generator,\n    validation_steps=validation_steps, \n    class_weight=class_weights)\n\nelapsed_time = time.time() - start_time\nelapsed_time = time.strftime(\"%H:%M:%S\", time.gmtime(elapsed_time))\n\nprint(\"\\nElapsed Time: \" + elapsed_time)\nprint(\"Completed Model Trainning\", date_time(1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 10. Model Performance \nModel Performance  Visualization over the Epochs"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_performance(history=history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n \n    ticksize = 18\n    titlesize = ticksize + 8\n    labelsize = ticksize + 5\n\n    figsize = (18, 5)\n    params = {'figure.figsize' : figsize,\n              'axes.labelsize' : labelsize,\n              'axes.titlesize' : titlesize,\n              'xtick.labelsize': ticksize,\n              'ytick.labelsize': ticksize}\n\n    plt.rcParams.update(params)\n    \n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    \n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n    \n    plt.show()\n    \n    \n\n\n\n# Predict the values from the validation dataset\nY_pred = model.predict(validation_generator)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(validation_generator.y,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes = range(10)) \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c09d411cfaed347995ec8308be13f2d848efe8","trusted":true},"cell_type":"code","source":"ypreds = model.predict_generator(generator=test_generator, steps = len(test_generator),  verbose=1)\n# ypreds","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f50d50f3e253212ed1083713ff6dcb16e893001","trusted":true},"cell_type":"code","source":"# ypred = ypreds[:,1]#\nypred = np.argmax(ypreds, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df = pd.read_csv(input_directory + \"sample_submission.csv\")\nsample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_gen_id = test_generator.index_array\nsample_submission_id = sample_df[\"ImageId\"]\n\nlen(test_gen_id), len(sample_submission_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_generator.index_array","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_list = list(sample_df.ImageId)\n\nids = [x+1 for x in test_generator.index_array]\n\n\npred_dict = dict((key, value) for (key, value) in zip(ids, ypred))\n\npred_list_new = [pred_dict[f] for f in sample_list]\n\ntest_df = pd.DataFrame({'ImageId': sample_list,'Label': pred_list_new})\n\ntest_df.to_csv('submission.csv', header=True, index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"64d369c0ed77ca5996c00c9b4ff23e8a31ba290b","trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}