{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CIFAR-10 - Object Recognition in Images","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:35:59.740727Z","iopub.execute_input":"2022-10-07T10:35:59.741803Z","iopub.status.idle":"2022-10-07T10:36:00.878163Z","shell.execute_reply.started":"2022-10-07T10:35:59.741674Z","shell.execute_reply":"2022-10-07T10:36:00.877095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:00.879989Z","iopub.execute_input":"2022-10-07T10:36:00.880494Z","iopub.status.idle":"2022-10-07T10:36:00.887311Z","shell.execute_reply.started":"2022-10-07T10:36:00.880463Z","shell.execute_reply":"2022-10-07T10:36:00.886291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.datasets import cifar10\n(X_train, Y_train), (X_test, Y_test) = cifar10.load_data()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:00.888657Z","iopub.execute_input":"2022-10-07T10:36:00.889135Z","iopub.status.idle":"2022-10-07T10:36:23.822178Z","shell.execute_reply.started":"2022-10-07T10:36:00.889099Z","shell.execute_reply":"2022-10-07T10:36:23.820860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('X_test.shape   :', X_test.shape)\nprint('Y_test.shape   :', Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:23.823482Z","iopub.execute_input":"2022-10-07T10:36:23.824217Z","iopub.status.idle":"2022-10-07T10:36:23.831153Z","shell.execute_reply.started":"2022-10-07T10:36:23.824177Z","shell.execute_reply":"2022-10-07T10:36:23.829982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def print_long_list(x, n_items=7, str_len=15):\n    print('len:', len(x))\n    for i in range(0, len(x), n_items):\n        for j in x[i:min(i+n_items, len(x))]:\n            print(str('{0: <'+str(str_len)+'}').format(str(j)), end=' ')\n        print()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:23.834428Z","iopub.execute_input":"2022-10-07T10:36:23.835022Z","iopub.status.idle":"2022-10-07T10:36:23.843789Z","shell.execute_reply.started":"2022-10-07T10:36:23.834969Z","shell.execute_reply":"2022-10-07T10:36:23.842211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_datatypes(df):\n    dtypes_col, dtypes_cnt = {}, {}\n    for col in df:\n        x = str(df[col].dtype)\n        if x not in dtypes_col:\n            dtypes_col[x] = [col]\n            dtypes_cnt[x] = 1\n        else:\n            dtypes_col[x] += [col]\n            dtypes_cnt[x] += 1\n    return dtypes_cnt, dtypes_col","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:23.845652Z","iopub.execute_input":"2022-10-07T10:36:23.846722Z","iopub.status.idle":"2022-10-07T10:36:23.854996Z","shell.execute_reply.started":"2022-10-07T10:36:23.846674Z","shell.execute_reply":"2022-10-07T10:36:23.853735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(X_train)\ndtypes_cnt","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:23.856482Z","iopub.execute_input":"2022-10-07T10:36:23.856852Z","iopub.status.idle":"2022-10-07T10:36:54.272021Z","shell.execute_reply.started":"2022-10-07T10:36:23.856819Z","shell.execute_reply":"2022-10-07T10:36:54.270900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(X_test)\ndtypes_cnt","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:36:54.273572Z","iopub.execute_input":"2022-10-07T10:36:54.273976Z","iopub.status.idle":"2022-10-07T10:37:00.302616Z","shell.execute_reply.started":"2022-10-07T10:36:54.273939Z","shell.execute_reply":"2022-10-07T10:37:00.301521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(Y_train)\ndtypes_cnt","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:00.303950Z","iopub.execute_input":"2022-10-07T10:37:00.304444Z","iopub.status.idle":"2022-10-07T10:37:00.773548Z","shell.execute_reply.started":"2022-10-07T10:37:00.304406Z","shell.execute_reply":"2022-10-07T10:37:00.772431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtypes_cnt, dtypes_col = get_datatypes(Y_test)\ndtypes_cnt","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:00.775127Z","iopub.execute_input":"2022-10-07T10:37:00.775475Z","iopub.status.idle":"2022-10-07T10:37:00.955934Z","shell.execute_reply.started":"2022-10-07T10:37:00.775445Z","shell.execute_reply":"2022-10-07T10:37:00.954754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(1,2,figsize=(24,6)) \n# Count plot for training set\nsns.countplot(Y_train.ravel(), ax=axs[0])\naxs[0].set_title('Distribution of training data')\naxs[0].set_xlabel('Classes')\n# Count plot for testing set\nsns.countplot(Y_test.ravel(), ax=axs[1])\naxs[1].set_title('Distribution of Testing data')\naxs[1].set_xlabel('Classes')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:00.957802Z","iopub.execute_input":"2022-10-07T10:37:00.958543Z","iopub.status.idle":"2022-10-07T10:37:01.347260Z","shell.execute_reply.started":"2022-10-07T10:37:00.958494Z","shell.execute_reply":"2022-10-07T10:37:01.345836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('X_test.shape   :', X_test.shape)\nprint('Y_test.shape   :', Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:01.348800Z","iopub.execute_input":"2022-10-07T10:37:01.349165Z","iopub.status.idle":"2022-10-07T10:37:01.355255Z","shell.execute_reply.started":"2022-10-07T10:37:01.349132Z","shell.execute_reply":"2022-10-07T10:37:01.353829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train / 255.0\nX_test  = X_test  / 255.0","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:01.357151Z","iopub.execute_input":"2022-10-07T10:37:01.357615Z","iopub.status.idle":"2022-10-07T10:37:01.864443Z","shell.execute_reply.started":"2022-10-07T10:37:01.357565Z","shell.execute_reply":"2022-10-07T10:37:01.863454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train.reshape(-1, 32, 32, 3)\nX_test  = X_test.reshape(-1, 32, 32, 3)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:01.869356Z","iopub.execute_input":"2022-10-07T10:37:01.869722Z","iopub.status.idle":"2022-10-07T10:37:01.875093Z","shell.execute_reply.started":"2022-10-07T10:37:01.869687Z","shell.execute_reply":"2022-10-07T10:37:01.873949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('X_test.shape   :', X_test.shape)\nprint('Y_test.shape   :', Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:01.876911Z","iopub.execute_input":"2022-10-07T10:37:01.877250Z","iopub.status.idle":"2022-10-07T10:37:01.889501Z","shell.execute_reply.started":"2022-10-07T10:37:01.877207Z","shell.execute_reply":"2022-10-07T10:37:01.888465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install visualkeras","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:01.890583Z","iopub.execute_input":"2022-10-07T10:37:01.890900Z","iopub.status.idle":"2022-10-07T10:37:17.578983Z","shell.execute_reply.started":"2022-10-07T10:37:01.890870Z","shell.execute_reply":"2022-10-07T10:37:17.577807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import visualkeras\nfrom sklearn.model_selection              import train_test_split\nfrom sklearn.metrics                      import confusion_matrix\nfrom tensorflow.keras.utils               import to_categorical\nfrom tensorflow.keras.models              import Sequential\nfrom tensorflow.keras.layers              import (Dense, Dropout, Flatten, Conv2D, MaxPool2D,\n                                                  Lambda, BatchNormalization)\nfrom tensorflow.keras.optimizers          import RMSprop\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks           import ReduceLROnPlateau\nfrom tensorflow.keras.optimizers          import RMSprop, Adam, SGD\nfrom tensorflow.keras.utils               import plot_model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:17.580580Z","iopub.execute_input":"2022-10-07T10:37:17.581044Z","iopub.status.idle":"2022-10-07T10:37:17.747317Z","shell.execute_reply.started":"2022-10-07T10:37:17.581000Z","shell.execute_reply":"2022-10-07T10:37:17.746145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_gen = ImageDataGenerator(\n                zoom_range = 0.1,\n                width_shift_range=0.1,\n                height_shift_range=0.1,\n                horizontal_flip=True,\n            )\ndata_gen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:17.749030Z","iopub.execute_input":"2022-10-07T10:37:17.749404Z","iopub.status.idle":"2022-10-07T10:37:18.341026Z","shell.execute_reply.started":"2022-10-07T10:37:17.749370Z","shell.execute_reply":"2022-10-07T10:37:18.340104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('X_test.shape   :', X_test.shape)\nprint('Y_test.shape   :', Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.342311Z","iopub.execute_input":"2022-10-07T10:37:18.342999Z","iopub.status.idle":"2022-10-07T10:37:18.350271Z","shell.execute_reply.started":"2022-10-07T10:37:18.342962Z","shell.execute_reply":"2022-10-07T10:37:18.348839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = to_categorical(Y_train, 10)\nY_test  = to_categorical(Y_test,  10)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.351933Z","iopub.execute_input":"2022-10-07T10:37:18.352769Z","iopub.status.idle":"2022-10-07T10:37:18.366343Z","shell.execute_reply.started":"2022-10-07T10:37:18.352701Z","shell.execute_reply":"2022-10-07T10:37:18.365104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('X_test.shape   :', X_test.shape)\nprint('Y_test.shape   :', Y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.368244Z","iopub.execute_input":"2022-10-07T10:37:18.368638Z","iopub.status.idle":"2022-10-07T10:37:18.376198Z","shell.execute_reply.started":"2022-10-07T10:37:18.368592Z","shell.execute_reply":"2022-10-07T10:37:18.374888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print_long_list(Y_train[:5], n_items=1)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.378494Z","iopub.execute_input":"2022-10-07T10:37:18.378923Z","iopub.status.idle":"2022-10-07T10:37:18.388176Z","shell.execute_reply.started":"2022-10-07T10:37:18.378886Z","shell.execute_reply":"2022-10-07T10:37:18.387003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.389976Z","iopub.execute_input":"2022-10-07T10:37:18.390374Z","iopub.status.idle":"2022-10-07T10:37:18.787316Z","shell.execute_reply.started":"2022-10-07T10:37:18.390329Z","shell.execute_reply":"2022-10-07T10:37:18.786401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train.shape  :', X_train.shape)\nprint('X_val.shape    :', X_val.shape)\nprint('Y_train.shape  :', Y_train.shape)\nprint('Y_val.shape    :', Y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.788794Z","iopub.execute_input":"2022-10-07T10:37:18.789383Z","iopub.status.idle":"2022-10-07T10:37:18.795185Z","shell.execute_reply.started":"2022-10-07T10:37:18.789347Z","shell.execute_reply":"2022-10-07T10:37:18.794294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train[32])","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:18.796411Z","iopub.execute_input":"2022-10-07T10:37:18.797319Z","iopub.status.idle":"2022-10-07T10:37:19.031522Z","shell.execute_reply.started":"2022-10-07T10:37:18.797283Z","shell.execute_reply":"2022-10-07T10:37:19.030226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_px = X_train.mean().astype(np.float32)\nstd_px  = X_train.std().astype(np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.033129Z","iopub.execute_input":"2022-10-07T10:37:19.033485Z","iopub.status.idle":"2022-10-07T10:37:19.798096Z","shell.execute_reply.started":"2022-10-07T10:37:19.033453Z","shell.execute_reply":"2022-10-07T10:37:19.796814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['Airplane', 'Automobile', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']\ninput_shape = (32, 32, 3)\nkernel_size = (4, 4)\npool_size = (2, 2)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.799849Z","iopub.execute_input":"2022-10-07T10:37:19.800205Z","iopub.status.idle":"2022-10-07T10:37:19.806224Z","shell.execute_reply.started":"2022-10-07T10:37:19.800172Z","shell.execute_reply":"2022-10-07T10:37:19.804816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def linear_model(optimizer, input_shape=input_shape):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Flatten())\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.807778Z","iopub.execute_input":"2022-10-07T10:37:19.808168Z","iopub.status.idle":"2022-10-07T10:37:19.820223Z","shell.execute_reply.started":"2022-10-07T10:37:19.808133Z","shell.execute_reply":"2022-10-07T10:37:19.819289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fully_connected_nn_model(optimizer, input_shape=input_shape):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Flatten())\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy', \n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.821514Z","iopub.execute_input":"2022-10-07T10:37:19.822437Z","iopub.status.idle":"2022-10-07T10:37:19.833286Z","shell.execute_reply.started":"2022-10-07T10:37:19.822397Z","shell.execute_reply":"2022-10-07T10:37:19.832256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_model(optimizer, input_shape=input_shape, kernel_size=kernel_size, pool_size=pool_size):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Flatten())\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.835756Z","iopub.execute_input":"2022-10-07T10:37:19.836139Z","iopub.status.idle":"2022-10-07T10:37:19.849624Z","shell.execute_reply.started":"2022-10-07T10:37:19.836106Z","shell.execute_reply":"2022-10-07T10:37:19.848338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_with_dropout_model(optimizer, input_shape=input_shape, kernel_size=kernel_size, pool_size=pool_size):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Dropout(0.25))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Dropout(0.25))\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Dropout(0.5))\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Dropout(0.5))\n    model.add(Flatten())\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.851387Z","iopub.execute_input":"2022-10-07T10:37:19.854164Z","iopub.status.idle":"2022-10-07T10:37:19.869046Z","shell.execute_reply.started":"2022-10-07T10:37:19.854111Z","shell.execute_reply":"2022-10-07T10:37:19.867838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_with_batch_normalization_model(optimizer, input_shape=input_shape, kernel_size=kernel_size, pool_size=pool_size):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Flatten())\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.870886Z","iopub.execute_input":"2022-10-07T10:37:19.871368Z","iopub.status.idle":"2022-10-07T10:37:19.887460Z","shell.execute_reply.started":"2022-10-07T10:37:19.871322Z","shell.execute_reply":"2022-10-07T10:37:19.886118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_nn_with_batch_normalization_with_dropout_model(optimizer, input_shape=input_shape, kernel_size=kernel_size, pool_size=pool_size):\n    model= Sequential()\n    model.add(Lambda(lambda x: (x-mean_px)/std_px, input_shape=input_shape))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Dropout(0.25))\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=32, kernel_size=kernel_size, padding='valid', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Dropout(0.25))\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Dropout(0.5))\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(Conv2D(filters=64, kernel_size=kernel_size, padding='same', activation='relu')),\n    model.add(BatchNormalization(axis=1)),\n    model.add(MaxPool2D(pool_size=pool_size)),\n    model.add(Dropout(0.5))\n    model.add(Flatten())\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=512, activation='relu'))\n    model.add(Dense(units=10, activation='softmax'))\n    model.compile(optimizer=optimizer,\n                  loss='categorical_crossentropy',\n                  metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.889239Z","iopub.execute_input":"2022-10-07T10:37:19.890023Z","iopub.status.idle":"2022-10-07T10:37:19.906610Z","shell.execute_reply.started":"2022-10-07T10:37:19.889984Z","shell.execute_reply":"2022-10-07T10:37:19.905420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fit_model(model, lr_reduction, epochs=10, batch_size=100):\n    batches = data_gen.flow(X_train,Y_train, batch_size=batch_size)\n    history = model.fit_generator(generator = batches,\n                                  epochs = epochs,\n                                  validation_data = (X_val,Y_val),\n                                  steps_per_epoch = batch_size,\n                                  validation_steps = batch_size,\n                                  callbacks = [lr_reduction],\n                                 )\n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.908033Z","iopub.execute_input":"2022-10-07T10:37:19.908849Z","iopub.status.idle":"2022-10-07T10:37:19.923282Z","shell.execute_reply.started":"2022-10-07T10:37:19.908801Z","shell.execute_reply":"2022-10-07T10:37:19.921835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def apply_learning_algorithm(model):\n    lr_reduction = ReduceLROnPlateau(monitor='val_acc', patience=2, factor=0.5, min_lr=0.00001)\n    model, history = fit_model(model, lr_reduction, epochs=50, batch_size=100)\n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.925015Z","iopub.execute_input":"2022-10-07T10:37:19.925363Z","iopub.status.idle":"2022-10-07T10:37:19.938687Z","shell.execute_reply.started":"2022-10-07T10:37:19.925332Z","shell.execute_reply":"2022-10-07T10:37:19.937596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [\n    linear_model(RMSprop(learning_rate=0.001)),\n    linear_model(Adam(learning_rate=0.001)),\n    fully_connected_nn_model(RMSprop(learning_rate=0.001)),\n    fully_connected_nn_model(Adam(learning_rate=0.001)),\n    conv_nn_model(RMSprop(learning_rate=0.001)),\n    conv_nn_model(Adam(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_model(RMSprop(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_model(Adam(learning_rate=0.001)),\n    conv_nn_with_dropout_model(RMSprop(learning_rate=0.001)),\n    conv_nn_with_dropout_model(Adam(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_with_dropout_model(RMSprop(learning_rate=0.001)),\n    conv_nn_with_batch_normalization_with_dropout_model(Adam(learning_rate=0.001)),\n]","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:19.940091Z","iopub.execute_input":"2022-10-07T10:37:19.941281Z","iopub.status.idle":"2022-10-07T10:37:21.718699Z","shell.execute_reply.started":"2022-10-07T10:37:19.941242Z","shell.execute_reply":"2022-10-07T10:37:21.717388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names = [\n    'linear_model\\n(RMSprop(lr=0.001))',\n    'linear_model\\n(Adam(lr=0.001))',\n    'fully_connected_nn_model\\n(RMSprop(lr=0.001))',\n    'fully_connected_nn_model\\n(Adam(lr=0.001))',\n    'conv_nn_model\\n(RMSprop(lr=0.001))',\n    'conv_nn_model\\n(Adam(lr=0.001))',\n    'conv_nn_with_batch_normalization_model\\n(RMSprop(lr=0.001))',\n    'conv_nn_with_batch_normalization_model\\n(Adam(lr=0.001))',\n    'conv_nn_with_dropout_model\\n(RMSprop(lr=0.001))',\n    'conv_nn_with_dropout_model\\n(Adam(lr=0.001))',\n    'conv_nn_with_batch_normalization_with_dropout_model\\n(RMSprop(lr=0.001))',\n    'conv_nn_with_batch_normalization_with_dropout_model\\n(Adam(lr=0.001))',\n]","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:37:21.720572Z","iopub.execute_input":"2022-10-07T10:37:21.720977Z","iopub.status.idle":"2022-10-07T10:37:21.727636Z","shell.execute_reply.started":"2022-10-07T10:37:21.720940Z","shell.execute_reply":"2022-10-07T10:37:21.726416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_history, all_models = [], []\nfor model in models:\n    fitted_model, history = apply_learning_algorithm(model)\n    all_models += [fitted_model]\n    all_history += [history]","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-10-07T10:37:21.729782Z","iopub.execute_input":"2022-10-07T10:37:21.730500Z","iopub.status.idle":"2022-10-07T10:42:02.456136Z","shell.execute_reply.started":"2022-10-07T10:37:21.730461Z","shell.execute_reply":"2022-10-07T10:42:02.455106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_acc = pd.DataFrame(model_names, columns = ['NN Model'])\nfinal_acc['Accuracy'] = [max(history.history['accuracy']) for history in all_history]\nfinal_acc","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:02.457856Z","iopub.execute_input":"2022-10-07T10:42:02.459019Z","iopub.status.idle":"2022-10-07T10:42:02.482034Z","shell.execute_reply.started":"2022-10-07T10:42:02.458980Z","shell.execute_reply":"2022-10-07T10:42:02.480808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12,6))\nsns.barplot(final_acc['NN Model'], final_acc['Accuracy'])\nplt.xlabel('Model',   fontsize = 12)\nplt.ylabel('Accuracy', fontsize = 12)\nplt.ylim(0.3, 1.0)\nplt.xticks(rotation=90)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:02.483560Z","iopub.execute_input":"2022-10-07T10:42:02.484640Z","iopub.status.idle":"2022-10-07T10:42:02.684853Z","shell.execute_reply.started":"2022-10-07T10:42:02.484603Z","shell.execute_reply":"2022-10-07T10:42:02.683561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model_name = final_acc.sort_values(by=['Accuracy']).tail(1)['NN Model'].iloc[0]\nbest_model = all_models[model_names.index(best_model_name)]\nbest_history = all_history[model_names.index(best_model_name)]\nprint(best_model_name, best_model)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:02.692088Z","iopub.execute_input":"2022-10-07T10:42:02.692469Z","iopub.status.idle":"2022-10-07T10:42:02.701904Z","shell.execute_reply.started":"2022-10-07T10:42:02.692438Z","shell.execute_reply":"2022-10-07T10:42:02.700672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:02.703854Z","iopub.execute_input":"2022-10-07T10:42:02.704360Z","iopub.status.idle":"2022-10-07T10:42:02.712935Z","shell.execute_reply.started":"2022-10-07T10:42:02.704314Z","shell.execute_reply":"2022-10-07T10:42:02.711619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model(model, show_shapes=True, show_layer_names=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-10-07T10:42:02.713942Z","iopub.execute_input":"2022-10-07T10:42:02.714335Z","iopub.status.idle":"2022-10-07T10:42:03.847989Z","shell.execute_reply.started":"2022-10-07T10:42:02.714301Z","shell.execute_reply":"2022-10-07T10:42:03.846545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"visualkeras.layered_view(model, legend=True)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:03.850543Z","iopub.execute_input":"2022-10-07T10:42:03.851066Z","iopub.status.idle":"2022-10-07T10:42:03.873679Z","shell.execute_reply.started":"2022-10-07T10:42:03.851011Z","shell.execute_reply":"2022-10-07T10:42:03.872498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_histpry_details(history):\n    for i in history.history:\n        print('===>', i)\n        vals = [round(x, 3) for x in history.history[i]]\n        print_long_list(vals, n_items=10, str_len=9)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:03.875601Z","iopub.execute_input":"2022-10-07T10:42:03.876128Z","iopub.status.idle":"2022-10-07T10:42:03.883390Z","shell.execute_reply.started":"2022-10-07T10:42:03.876075Z","shell.execute_reply":"2022-10-07T10:42:03.882032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_history(history, fig_title, shape=(24,6)):\n    fig, ax = plt.subplots(1,2, figsize=shape)\n    fig.suptitle(fig_title, fontsize=16)\n    ax[0].plot(history.history['loss'],     color='b', label=\"Training loss\")\n    ax[0].plot(history.history['val_loss'], color='r', label=\"validation loss\",axes =ax[0])\n    legend = ax[0].legend(loc='best', shadow=True)\n    ax[0].set_xlabel('Epochs')\n    ax[0].set_ylabel('Loss')\n    ax[1].plot(history.history['accuracy'],     color='b', label=\"Training accuracy\")\n    ax[1].plot(history.history['val_accuracy'], color='r', label=\"Validation accuracy\")\n    legend = ax[1].legend(loc='best', shadow=True)\n    ax[1].set_xlabel('Epochs')\n    ax[1].set_ylabel('Accuracy')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:03.885220Z","iopub.execute_input":"2022-10-07T10:42:03.886153Z","iopub.status.idle":"2022-10-07T10:42:03.897197Z","shell.execute_reply.started":"2022-10-07T10:42:03.886072Z","shell.execute_reply":"2022-10-07T10:42:03.895789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_histpry_details(best_history)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:03.899336Z","iopub.execute_input":"2022-10-07T10:42:03.899692Z","iopub.status.idle":"2022-10-07T10:42:03.916495Z","shell.execute_reply.started":"2022-10-07T10:42:03.899659Z","shell.execute_reply":"2022-10-07T10:42:03.915181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_history(best_history, 'History of the Model')","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:03.918814Z","iopub.execute_input":"2022-10-07T10:42:03.919360Z","iopub.status.idle":"2022-10-07T10:42:04.337710Z","shell.execute_reply.started":"2022-10-07T10:42:03.919314Z","shell.execute_reply":"2022-10-07T10:42:04.336882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(cm, shape=(10,10)):\n    f, ax = plt.subplots(figsize=shape)\n    _ = sns.heatmap(cm, \n                    square=True, \n                    cbar_kws={'shrink':.9 }, \n                    ax=ax,\n                    annot=True, \n                    linewidths=0.01,\n                    vmax=1.0, \n                    linecolor='white',\n                    fmt= '.0f',\n                    annot_kws={'fontsize':14},\n                    )\n    plt.title('Correlation of Prediction', y=1.05, size=15)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:04.339239Z","iopub.execute_input":"2022-10-07T10:42:04.339890Z","iopub.status.idle":"2022-10-07T10:42:04.346211Z","shell.execute_reply.started":"2022-10-07T10:42:04.339854Z","shell.execute_reply":"2022-10-07T10:42:04.345215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred = best_model.predict(X_val)\nY_pred_classes = np.argmax(Y_pred,axis = 1) \nY_true = np.argmax(Y_val,axis = 1) \ncm = confusion_matrix(Y_true, Y_pred_classes)\nplot_confusion_matrix(cm, shape=(10,10))","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:04.347311Z","iopub.execute_input":"2022-10-07T10:42:04.347614Z","iopub.status.idle":"2022-10-07T10:42:06.675791Z","shell.execute_reply.started":"2022-10-07T10:42:04.347587Z","shell.execute_reply":"2022-10-07T10:42:06.674510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_errors(sorted_dela_errors, img_errors, pred_errors, obs_errors):\n    i, nrows, ncols  = 0, 4, 8\n    shape = (ncols*3, nrows*3.25)\n    errors_index = sorted_dela_errors[-nrows*ncols:]\n    fig, ax = plt.subplots(nrows, ncols, sharex=True, sharey=True, figsize=shape)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[i]\n            ax[row,col].imshow((img_errors[error]).reshape(input_shape))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(labels[pred_errors[error]],labels[obs_errors[error]]))\n            i += 1","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:06.677324Z","iopub.execute_input":"2022-10-07T10:42:06.677835Z","iopub.status.idle":"2022-10-07T10:42:06.686921Z","shell.execute_reply.started":"2022-10-07T10:42:06.677788Z","shell.execute_reply":"2022-10-07T10:42:06.685538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"errors = (Y_pred_classes - Y_true != 0)\nY_pred_classes_errors = Y_pred_classes[errors]\nY_pred_errors = Y_pred[errors]\nY_true_errors = Y_true[errors]\nX_val_errors = X_val[errors]","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:06.688832Z","iopub.execute_input":"2022-10-07T10:42:06.689647Z","iopub.status.idle":"2022-10-07T10:42:06.731833Z","shell.execute_reply.started":"2022-10-07T10:42:06.689606Z","shell.execute_reply":"2022-10-07T10:42:06.730538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_errors_prob = np.max(Y_pred_errors, axis = 1)\ntrue_prob_errors = np.diagonal(np.take(Y_pred_errors, Y_true_errors, axis=1))\ndelta_pred_true_errors = Y_pred_errors_prob - true_prob_errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\ndisplay_errors(sorted_dela_errors, X_val_errors, Y_pred_classes_errors, Y_true_errors)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:42:06.734604Z","iopub.execute_input":"2022-10-07T10:42:06.735768Z","iopub.status.idle":"2022-10-07T10:42:10.738303Z","shell.execute_reply.started":"2022-10-07T10:42:06.735688Z","shell.execute_reply":"2022-10-07T10:42:10.737032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = best_model.predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = [labels[i] for i in results]\nresults = pd.Series(results,name=\"Label\")","metadata":{"execution":{"iopub.status.busy":"2022-10-07T10:59:55.710912Z","iopub.execute_input":"2022-10-07T10:59:55.711942Z","iopub.status.idle":"2022-10-07T10:59:56.388224Z","shell.execute_reply.started":"2022-10-07T10:59:55.711896Z","shell.execute_reply":"2022-10-07T10:59:56.386661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,len(X_test)+1), name = \"id\"),results],axis = 1)\nsubmission.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T11:00:30.879596Z","iopub.execute_input":"2022-10-07T11:00:30.880064Z","iopub.status.idle":"2022-10-07T11:00:30.893893Z","shell.execute_reply.started":"2022-10-07T11:00:30.879997Z","shell.execute_reply":"2022-10-07T11:00:30.892660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-07T11:00:36.246940Z","iopub.execute_input":"2022-10-07T11:00:36.248327Z","iopub.status.idle":"2022-10-07T11:00:36.269055Z","shell.execute_reply.started":"2022-10-07T11:00:36.248265Z","shell.execute_reply":"2022-10-07T11:00:36.267647Z"},"trusted":true},"execution_count":null,"outputs":[]}]}