{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:36.324047Z","iopub.execute_input":"2021-07-09T11:48:36.324422Z","iopub.status.idle":"2021-07-09T11:48:37.187384Z","shell.execute_reply.started":"2021-07-09T11:48:36.324340Z","shell.execute_reply":"2021-07-09T11:48:37.186484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Load Data","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:40.808110Z","iopub.execute_input":"2021-07-09T11:48:40.808451Z","iopub.status.idle":"2021-07-09T11:48:46.400806Z","shell.execute_reply.started":"2021-07-09T11:48:40.808421Z","shell.execute_reply":"2021-07-09T11:48:46.399926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Data preprocessing","metadata":{}},{"cell_type":"code","source":"def get_image(row):\n    '''\n    This function takes a series as input and returns image and it's label\n    '''\n    \n    label = row['label']\n    image = row.drop('label').values\n    image = image.reshape(28, 28)\n    image = np.array(image, np.float32)\n        \n    image = image / 255.0\n    \n    return image, label","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:46.402292Z","iopub.execute_input":"2021-07-09T11:48:46.402628Z","iopub.status.idle":"2021-07-09T11:48:46.408391Z","shell.execute_reply.started":"2021-07-09T11:48:46.402592Z","shell.execute_reply":"2021-07-09T11:48:46.407208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_images():\n    '''\n    This function is used to visualize the images \n    '''\n    plt.figure(figsize=(20, 20))\n    for i in range(25):\n        plt.subplot(5, 5, i+1)\n        index = np.random.randint(0, len(df_train))\n        image, label = get_image(df_train.iloc[index])\n        \n        plt.imshow(image)\n        plt.title('Label: {}'.format(label))\n        \n    plt.show()\n    \nshow_images()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:46.410357Z","iopub.execute_input":"2021-07-09T11:48:46.410745Z","iopub.status.idle":"2021-07-09T11:48:49.155736Z","shell.execute_reply.started":"2021-07-09T11:48:46.410706Z","shell.execute_reply":"2021-07-09T11:48:49.154712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nDefine Data generator class\n'''\nfrom tensorflow.keras.utils import Sequence\nclass DataGenerator(Sequence):\n    'Generates data for Keras'\n    def __init__(self, name, images, labels, batch_size=32, shuffle=True):\n        \n        \n        'Initialization'\n        self.name = name\n        self.batch_size = batch_size\n        self.images = images\n        self.labels = labels\n        self.shuffle = shuffle\n        self.on_epoch_end()\n\n    def __len__(self):\n        'Denotes the number of batches per epoch'\n        #print('returning length')\n        #print(int(np.floor(len(self.sentences) / self.batch_size)))\n        return int(np.floor(len(self.labels) / self.batch_size))\n\n    def __getitem__(self, index):\n        'Generate one batch of data'\n        # Generate indexes of the batch\n        #print(index)\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n\n        # Find list of IDs\n        images = [self.images[k] for k in indexes]\n        labels = [self.labels[k] for k in indexes]\n        \n        images = np.array(images)\n        labels = np.array(labels)\n        \n        return images, labels\n\n    def on_epoch_end(self):\n        'Updates indexes after each epoch'\n        self.indexes = np.arange(len(self.labels))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n\n    def add_data(self, images, labels):\n        self.images=np.append(self.images, images, axis=0)\n        self.labels=np.append(self.labels, labels, axis=0)\n        self.on_epoch_end()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:49.157081Z","iopub.execute_input":"2021-07-09T11:48:49.157450Z","iopub.status.idle":"2021-07-09T11:48:55.424456Z","shell.execute_reply.started":"2021-07-09T11:48:49.157410Z","shell.execute_reply":"2021-07-09T11:48:55.423466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_images(df, generator_name, batch_size=32):\n    \n    total_images = len(df)\n    num_batches = total_images // batch_size\n    batch_count = 0\n    while batch_count < num_batches:\n        x_batch = np.zeros((batch_size, 28, 28, 1))\n        label_batch = np.zeros((batch_size,))\n\n        for i in range(batch_size):\n            index = np.random.randint(0, total_images)\n            image, label = get_image(df.iloc[index])\n            image = image.reshape(28, 28, 1)\n\n            x_batch[i] = image \n            label_batch[i] = label\n            \n        if batch_count == 0:\n            generator = DataGenerator(generator_name, x_batch, label_batch)\n        else:\n            generator.add_data(x_batch, label_batch)\n            \n        batch_count += 1\n\n    return generator","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:55.426441Z","iopub.execute_input":"2021-07-09T11:48:55.426798Z","iopub.status.idle":"2021-07-09T11:48:55.435205Z","shell.execute_reply.started":"2021-07-09T11:48:55.426764Z","shell.execute_reply":"2021-07-09T11:48:55.434151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Defining training and validation generators\ndf_val = df_train.sample(4000)\ndf_train = df_train.sample(len(df_train)-4000)\n\ntrain_gen = generate_images(df_train, 'training_generator')\nval_gen = generate_images(df_val, 'validation generator')","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:48:55.437246Z","iopub.execute_input":"2021-07-09T11:48:55.438005Z","iopub.status.idle":"2021-07-09T11:50:17.116385Z","shell.execute_reply.started":"2021-07-09T11:48:55.437961Z","shell.execute_reply":"2021-07-09T11:50:17.115390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Define Model","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import Convolution2D, MaxPool2D, MaxPooling2D, LeakyReLU, Flatten, Dense, BatchNormalization, Dropout, Concatenate, Activation, Input, Add\nfrom tensorflow.keras.models import Model\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:50:17.118191Z","iopub.execute_input":"2021-07-09T11:50:17.118601Z","iopub.status.idle":"2021-07-09T11:50:17.124086Z","shell.execute_reply.started":"2021-07-09T11:50:17.118558Z","shell.execute_reply":"2021-07-09T11:50:17.122693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input = Input(shape=(28, 28, 1), name='input')\n\nconv_1 = Convolution2D(32, 3, padding='same', activation='relu')(input)\nbatch_1 = BatchNormalization(name='batch_1')(conv_1)\nconv_2 = Convolution2D(32, 3, padding='same', activation='relu')(conv_1)\nbatch_2 = BatchNormalization(name='batch_2')(conv_2)\n\npool_1 = MaxPooling2D(pool_size=(2, 2))(batch_2)\ndrop_1 = Dropout(0.4)(pool_1)\n\nconv_3 = Convolution2D(64, 3, padding='same', activation='relu')(drop_1)\nbatch_3 = BatchNormalization(name='batch_3')(conv_3)\nconv_4 = Convolution2D(64, 3, padding='same', activation='relu')(conv_3)\nbatch_4 = BatchNormalization(name='batch_4')(conv_4)\ndrop_2 = Dropout(0.4)(batch_4)\n\nadd_1 = Concatenate()([pool_1, drop_2])\npool_2 = MaxPooling2D(pool_size=(2, 2))(add_1)\nbatch_5 = BatchNormalization(name='batch_5')(pool_2)\n\nconv_5 = Convolution2D(128, 3, padding='same', activation='relu')(pool_2)\nbatch_6 = BatchNormalization(name='batch_6')(conv_5)\n\nflat_2 = Flatten(name='flat_2')(batch_6)\ndense_1 = Dense(128, name='dense_1')(flat_2)\nact_1 = LeakyReLU()(dense_1)\ndrop_3 = Dropout(0.4)(act_1)\nlabel = Dense(10, activation='softmax', name='label')(drop_3)\n\nmodel = Model(inputs=input, outputs=label)\nmodel.compile(\n    loss='sparse_categorical_crossentropy',\n    metrics=['accuracy'],\n    optimizer='adam'\n)\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:50:30.346689Z","iopub.execute_input":"2021-07-09T11:50:30.347242Z","iopub.status.idle":"2021-07-09T11:50:33.137611Z","shell.execute_reply.started":"2021-07-09T11:50:30.347194Z","shell.execute_reply":"2021-07-09T11:50:33.136706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Training ","metadata":{}},{"cell_type":"code","source":"class Logger(tf.keras.callbacks.Callback):\n    def on_epoch_end(self, epoch, logs=None):\n        print() #just for spacing between two epochs stats","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:50:36.072146Z","iopub.execute_input":"2021-07-09T11:50:36.072487Z","iopub.status.idle":"2021-07-09T11:50:36.079896Z","shell.execute_reply.started":"2021-07-09T11:50:36.072457Z","shell.execute_reply":"2021-07-09T11:50:36.078930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkpoints = tf.keras.callbacks.ModelCheckpoint(\n    './models/checkpoint.h5',\n    monitor=\"val_loss\",\n    save_best_only=True,\n    verbose=1\n)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:50:37.032137Z","iopub.execute_input":"2021-07-09T11:50:37.032488Z","iopub.status.idle":"2021-07-09T11:50:37.039385Z","shell.execute_reply.started":"2021-07-09T11:50:37.032456Z","shell.execute_reply":"2021-07-09T11:50:37.038274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(train_gen, \n              validation_data=val_gen, \n              epochs=20,\n              callbacks=[Logger(), tf.keras.callbacks.TensorBoard(log_dir='./logs'), checkpoints],\n              verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:50:38.901269Z","iopub.execute_input":"2021-07-09T11:50:38.901595Z","iopub.status.idle":"2021-07-09T11:55:20.249961Z","shell.execute_reply.started":"2021-07-09T11:50:38.901559Z","shell.execute_reply":"2021-07-09T11:55:20.249041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.save('./models/digit_recognizer_skip.h5')\nmodel.load_weights('./models/checkpoint.h5')\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:55:26.721039Z","iopub.execute_input":"2021-07-09T11:55:26.721434Z","iopub.status.idle":"2021-07-09T11:55:26.766287Z","shell.execute_reply.started":"2021-07-09T11:55:26.721396Z","shell.execute_reply":"2021-07-09T11:55:26.765393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Test Data","metadata":{}},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:55:31.447132Z","iopub.execute_input":"2021-07-09T11:55:31.447502Z","iopub.status.idle":"2021-07-09T11:55:31.469995Z","shell.execute_reply.started":"2021-07-09T11:55:31.447470Z","shell.execute_reply":"2021-07-09T11:55:31.468850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_test = df_test.values\nx_test = x_test.reshape(28000, 28, 28, 1) / 255.0\nprint(x_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:55:32.510125Z","iopub.execute_input":"2021-07-09T11:55:32.510458Z","iopub.status.idle":"2021-07-09T11:55:32.557553Z","shell.execute_reply.started":"2021-07-09T11:55:32.510428Z","shell.execute_reply":"2021-07-09T11:55:32.556464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Make predictions\npredictions = model.predict(x_test)\ny_pred = []\nfor pred in predictions:\n    y_pred.append(pred.argmax())\n    \nprint(len(y_pred))","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:55:33.839184Z","iopub.execute_input":"2021-07-09T11:55:33.839530Z","iopub.status.idle":"2021-07-09T11:55:39.517235Z","shell.execute_reply.started":"2021-07-09T11:55:33.839501Z","shell.execute_reply":"2021-07-09T11:55:39.516335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\nsubmission['Label']=y_pred\nsubmission.to_csv('solution_3.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-09T11:55:45.690486Z","iopub.execute_input":"2021-07-09T11:55:45.690822Z","iopub.status.idle":"2021-07-09T11:55:46.075373Z","shell.execute_reply.started":"2021-07-09T11:55:45.690793Z","shell.execute_reply":"2021-07-09T11:55:46.074426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}