{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook features Tensorflow **tf.data API** and **custom Keras classes** for the model. \n\n## This notebook implements Kfold validation with tf.data.dataset\n\n***You like it please upvote :)***"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport os\nimport datetime\nimport time\nimport pandas as pd\nfrom PIL import Image\nimport random\nimport glob\nfrom matplotlib import pyplot as plt\nimport seaborn as sn\nimport re\nimport tensorflow as tf\nimport tensorflow.keras.preprocessing as pp\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau,EarlyStopping\nimport tensorflow_addons as tfa\nfrom imgaug import augmenters as iaa\n\nfrom skimage import transform\n%config IPCompleter.greedy = True\n%config InlineBackend.figure_format = 'retina'\n%matplotlib inline\npd.set_option('mode.chained_assignment', None)\nsn.set(font_scale=1.5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['label']\nX = train.drop('label',axis=1).values\ntest = test.values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"RANDOM_STATE = 75\n\ndef random_seed(seed):\n    random.seed(RANDOM_STATE)\n    os.environ['PYTHONHASHSEED'] = str(RANDOM_STATE)\n    np.random.seed(RANDOM_STATE)\n    tf.random.set_seed(RANDOM_STATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_image(image):\n    plt.rcParams[\"axes.grid\"] = False\n    plt.axis('off')\n    plt.imshow(image)   \n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTOTUNE = tf.data.experimental.AUTOTUNE\nEPOCHS=50\nSHUFFLE_BUFFER_SIZE=5000 \nBATCH_SIZE=64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef reshape(image, label):\n    return tf.reshape(image,(28,28,1)),label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef normalize(image, label):\n    image = tf.cast(image, tf.float32)\n    return image/255.0, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef rotate_tf(image):\n    #https://github.com/tensorflow/tensorflow/issues/30112#issuecomment-549330939\n    random_angles = tf.random.uniform(shape = (), minval = -np\n    .pi / 15, maxval = np.pi / 15)\n    return tfa.image.rotate(image,random_angles)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"@tf.function\ndef translate_tf(image):\n    ratio=tf.random.uniform((2,), minval=-5, maxval=5, dtype=tf.dtypes.int32)\n    ratio=tf.cast(ratio, tf.dtypes.float32)\n    return tfa.image.translate(image, ratio,'BILINEAR')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def augment(image, label):\n    image = rotate_tf(image)\n    image = translate_tf(image)\n    return image, label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_data_set(images, labels, is_training=False):\n    dataset = tf.data.Dataset.from_tensor_slices((images,labels))\n    dataset = dataset.map(reshape, num_parallel_calls=AUTOTUNE)\n    dataset = dataset.map(normalize, num_parallel_calls=AUTOTUNE)\n    \n    if is_training:\n        dataset = dataset.map(augment, num_parallel_calls=AUTOTUNE)\n        \n    dataset = dataset.shuffle(SHUFFLE_BUFFER_SIZE)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTOTUNE)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class CNNBlock(layers.Layer):\n    def __init__(self, out_channel, kernel_size=3, dropout=None):\n        super(CNNBlock,self).__init__()\n        self.conv = layers.Conv2D(out_channel, kernel_size,kernel_initializer='he_normal', padding=\"same\")\n        self.bn = layers.BatchNormalization()\n        self.maxpool = layers.MaxPool2D(pool_size=(2,2))\n        self.dropout=None\n        if dropout:\n            self.dropout = layers.Dropout(dropout)\n    \n    def call(self, input_tensor, training=False):\n        x = self.conv(input_tensor)\n        x = self.bn(x, training=training)\n        x = tf.nn.relu(x)\n        x = self.maxpool(x)\n        if self.dropout:\n            x = self.dropout(x)\n        return x    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.tensorflow.org/guide/keras/train_and_evaluate\ndef get_uncompiled_model():\n    model = keras.Sequential(\n     [CNNBlock(32), CNNBlock(64,dropout=.25), CNNBlock(128, dropout=.25), CNNBlock(256, dropout=.25), layers.Flatten(), layers.Dense(256, activation = \"relu\"),layers.Dropout(.5), layers.Dense(10)]\n)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_compiled_model():\n    model = get_uncompiled_model()\n    model.compile(\n        optimizer=\"rmsprop\",\n        loss=[keras.losses.SparseCategoricalCrossentropy(from_logits=True),],\n        metrics=[\"accuracy\"],\n    )\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test=np.zeros((test.shape[0],10))\n\nrun_history = {}\n\nfrom sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=3, random_state=RANDOM_STATE, shuffle=True)\n\nfor fold, (train_index, test_index) in enumerate(skf.split(X, y)):\n    print(\"Starting fold {}\".format(fold))\n\n    X_train, X_val = X[train_index], X[test_index]\n    \n    y_train, y_val = y[train_index], y[test_index]\n    \n    train_dataset = get_data_set(X_train, y_train, is_training=True)\n    test_dataset = get_data_set(X_val,y_val)\n\n    sample_x, sample_y = next(iter(train_dataset))\n    \n    print(\"Label:\",sample_y[fold])\n    show_image(sample_x[fold][:,:,0])\n    \n    model = get_compiled_model() \n    reduce_lr_on_plateau = ReduceLROnPlateau(monitor='val_loss',\n                       factor=0.5,\n                       patience=3,\n                       min_lr=0.0001,\n                           verbose=1)  \n    filepath = 'model_{}.h5'.format(fold)\n    checkpoint=tf.keras.callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True,\n            save_weights_only=True, mode='min')\n    callbacks_list = [reduce_lr_on_plateau,checkpoint]\n    \n    history = model.fit(train_dataset, epochs=EPOCHS, verbose=1,validation_data=test_dataset,callbacks=callbacks_list).history\n    \n    run_history[\"fold_{}\".format(fold)] = history\n    \n    print(\"Loading model...\")\n    model.load_weights(filepath)\n    \n    print('predict on test....')\n    preds=model.predict(test.reshape(-1,28,28,1),verbose=1)\n    \n    pred_test += preds\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_loss(column, plot_name):\n    plt.rcParams['figure.figsize'] = fig_size\n\n    for key, data in run_history.items():\n        plt.plot(data[column])\n\n    leg = plt.legend(run_history.keys(), loc='upper right')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.title(plot_name)\n    for line in leg.get_lines():\n        line.set_linewidth(10)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_loss(\"loss\",'Training Loss')\nplot_loss(\"val_loss\",'Validation Loss')\nplot_loss(\"accuracy\",'Training accuracy')\nplot_loss(\"val_accuracy\",'Validation accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = tf.argmax(pred_test,axis = 1)\n\nresults = pd.Series(results, name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_keras_classes_k_fold.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}