{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import PIL\nimport urllib\nimport numpy as np\nimport matplotlib.pyplot as plt\nplt.figure(figsize=(20,5))\nplt.axis(False)\nplt.imshow(\n    np.array(\n        PIL.Image.open(\n            urllib.request.urlopen(\n                \"https://user-images.githubusercontent.com/57945364/221378314-e46b5eaa-7507-4d75-9355-c0a61a5af4c4.png\"\n            )\n        )\n    )\n)","metadata":{"execution":{"iopub.status.busy":"2023-02-25T20:31:22.517244Z","iopub.execute_input":"2023-02-25T20:31:22.517961Z","iopub.status.idle":"2023-02-25T20:31:22.856411Z","shell.execute_reply.started":"2023-02-25T20:31:22.517922Z","shell.execute_reply":"2023-02-25T20:31:22.855276Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*CNN is a powerful algorithm for image processing*\n\nThis notebook provides the base to approach for image classification(here, for MNIST) problem. With data visualization, EDA, it takes the approach to experimenting and selecting optimized parameters for modeling under its hood. With sufficient and necessary(if needed) information, I wish the process will be helpful for you too.  \n\n# Index\n1. Imports\n2. EDA\n3. Understanding\n4. Approach to modeling\n5. Deep models","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom keras.utils.np_utils import to_categorical\nfrom matplotlib import cm\nfrom tensorflow.keras.layers import (\n    AvgPool2D,\n    BatchNormalization,\n    Conv2D,\n    Dense,\n    Dropout,\n    Flatten,\n    MaxPool2D,\n    MaxPooling2D,\n)\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:25.675588Z","iopub.execute_input":"2023-02-25T16:55:25.676008Z","iopub.status.idle":"2023-02-25T16:55:25.683684Z","shell.execute_reply.started":"2023-02-25T16:55:25.675971Z","shell.execute_reply":"2023-02-25T16:55:25.682633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nx = df.drop(columns=\"label\")\ny = df[\"label\"]","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:25.861698Z","iopub.execute_input":"2023-02-25T16:55:25.862224Z","iopub.status.idle":"2023-02-25T16:55:28.520361Z","shell.execute_reply.started":"2023-02-25T16:55:25.862149Z","shell.execute_reply":"2023-02-25T16:55:28.519113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:28.524950Z","iopub.execute_input":"2023-02-25T16:55:28.525355Z","iopub.status.idle":"2023-02-25T16:55:28.553459Z","shell.execute_reply.started":"2023-02-25T16:55:28.525318Z","shell.execute_reply":"2023-02-25T16:55:28.552161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:29.061932Z","iopub.execute_input":"2023-02-25T16:55:29.062711Z","iopub.status.idle":"2023-02-25T16:55:31.510072Z","shell.execute_reply.started":"2023-02-25T16:55:29.062672Z","shell.execute_reply":"2023-02-25T16:55:31.508838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distribution of the label count","metadata":{}},{"cell_type":"code","source":"def gradientbars(bars, ydata, cmap):\n    ax = bars[0].axes\n    lim = ax.get_xlim() + ax.get_ylim()\n    ax.axis(lim)\n    for bar in bars:\n        bar.set_facecolor(\"none\")\n        x, y = bar.get_xy()\n        w, h = bar.get_width(), bar.get_height()\n        grad = np.atleast_2d(np.linspace(0, 1 * h / max(ydata), 256)).T\n        ax.imshow(\n            grad,\n            extent=[x, x + w, y, y + h],\n            origin=\"lower\",\n            aspect=\"auto\",\n            norm=cm.colors.NoNorm(vmin=0, vmax=1),\n            cmap=plt.get_cmap(cmap),\n        )\n\n\nfig, ax = plt.subplots(figsize=(10, 5))\nmy_bar = ax.bar(\n    np.sort(df[\"label\"].unique()), df[\"label\"].value_counts().sort_index(), alpha=0.4\n)\nax.set_xlabel(\"Number\")\ngradientbars(my_bar, df[\"label\"].value_counts().sort_index(), \"RdPu\")\nplt.show()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-02-25T16:55:31.512398Z","iopub.execute_input":"2023-02-25T16:55:31.512880Z","iopub.status.idle":"2023-02-25T16:55:31.718114Z","shell.execute_reply.started":"2023-02-25T16:55:31.512833Z","shell.execute_reply":"2023-02-25T16:55:31.716849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Sample images","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 10, figsize=(20, 20))\nfor i in range(10):\n    ax[i].imshow(x.iloc[i].values.reshape(28, 28))\n    ax[i].set_xlabel(\"Class: \" + str(df[\"label\"].iloc[i]))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:32.004860Z","iopub.execute_input":"2023-02-25T16:55:32.005284Z","iopub.status.idle":"2023-02-25T16:55:32.991307Z","shell.execute_reply.started":"2023-02-25T16:55:32.005249Z","shell.execute_reply":"2023-02-25T16:55:32.989932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check for nulls","metadata":{}},{"cell_type":"code","source":"# isna() - returns full df with each value labelled\n# any() #1 any null in each column\n# any() #2 null amongst any columns(complete data)\ndf.isna().any().any()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:34.821550Z","iopub.execute_input":"2023-02-25T16:55:34.822031Z","iopub.status.idle":"2023-02-25T16:55:34.849383Z","shell.execute_reply.started":"2023-02-25T16:55:34.821985Z","shell.execute_reply":"2023-02-25T16:55:34.847669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get the usable reshaped size","metadata":{}},{"cell_type":"code","source":"x = np.array(x)\ny = np.array(y)\n\nx = x.reshape(x.shape[0], 28, 28, 1)\nprint(x.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:36.428322Z","iopub.execute_input":"2023-02-25T16:55:36.428777Z","iopub.status.idle":"2023-02-25T16:55:36.535049Z","shell.execute_reply.started":"2023-02-25T16:55:36.428741Z","shell.execute_reply":"2023-02-25T16:55:36.533513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One hot encoding of the target classes","metadata":{}},{"cell_type":"code","source":"y = to_categorical(y, num_classes=np.unique(y).size)\ny\n\nprint(y.shape)","metadata":{"execution":{"iopub.status.busy":"2023-02-25T16:55:37.604210Z","iopub.execute_input":"2023-02-25T16:55:37.604630Z","iopub.status.idle":"2023-02-25T16:55:37.613309Z","shell.execute_reply.started":"2023-02-25T16:55:37.604595Z","shell.execute_reply":"2023-02-25T16:55:37.612077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Understanding\n\n## Convolution Neural Networks\nConvolutional Neural Networks are very similar to ordinary Neural Networks. On a bigger scale, each neuron in the Neural Net, recives some inputs, performs dot product, and optionally lfows it through non-linearity. The whole network expresses the structure to having a raw image pixels as input and giving the class scores as the outputs. The outputs are still based on the loss functions like SVM/Softmax (Softmax Loss is actually just a Softmax Activation plus a Cross-Entropy Loss). \n\nCNNs are different from NNs as their arhcitectures make the explicit assumption that the inputs are images, which allows us to encode certain properties into the architecture. These then make the forward function more efficient to implement and vastly reduce the amount of parameters in the network.\n\n### But why not simply use NNs?\nRegular Neural Nets don’t scale well to full images. In CIFAR-10, images are only of size $32 \\times 32 \\times 3$ (32 wide, 32 high, 3 color channels), so a single fully-connected neuron in a first hidden layer of a regular Neural Network would have $32 * 32 * 3 = 3072$ weights. This amount still seems manageable, but clearly this fully-connected structure does not scale to larger images. For example, an image of more respectable size, e.g. $200 \\times 200 \\times 3$, would lead to neurons that have $200 * 200 * 3 = 120,000$ weights. Moreover, we would almost certainly want to have several such neurons, so the parameters would add up quickly! Clearly, this full connectivity is wasteful and the huge number of parameters would quickly lead to overfitting.\n\n### What is it about CNNs?\n\n3D volumes of neurons. Convolutional Neural Networks take advantage of the fact that the input consists of images and they constrain the architecture in a more sensible way. In particular, unlike a regular Neural Network, the layers of a ConvNet have neurons arranged in 3 dimensions: width, height, depth. the neurons in a layer will only be connected to a small region of the layer before it, instead of all of the neurons in a fully-connected manner. Moreover, the final output layer would have depth depending on the number of filters used.\nEg: for input of $28 \\times 28 \\times 1$ and 32 filters will give output of $28 \\times 28 \\times 12$. These are passed through the final layer, to give $1 \\times 1 \\times 10$ , these are the class of scores, the predictions \n\nIts important to have a small deviation to **pooling layer**.\n\nWhere Convolutional layers extract features from images, Pooling layers consolidate the features learned by CNNs. Its purpose is to gradually shrink the representation’s spatial dimension to minimize the number of parameters and computations in the network. Pooling layers are one of the building blocks of Convolutional Neural Networks.\n\n### Let's visualize!\n<img src = \"https://www.researchgate.net/publication/331717561/figure/fig1/AS:949173058826240@1603311900040/Regular-NN-and-CNN-a-An-example-of-regular-feedforward-neural-network-represented-by.png\" width = \"50%\"/>\n\n```Left```: A regular 3-layer NN.\n\n```Right```: A CNN arranges its neurons in three dimensions (width, height, depth), as visualized in one of the layers. Every layer of a CNN transforms the 3D input volume to a 3D output volume of neuron activations. \n\n### What are these filters?\nThe filters argument sets the number of convolutional filters in that layer. These filters are initialized to small, random values, using the method specified by the kernel_initializer argument. During network training, the filters are updated in a way that minimizes the loss. So over the course of training, the filters will learn to detect certain features, like edges and textures\n\n### Conv2 layer form tensorflow keras\nThis layer creates a convolution kernel that is convolved with the layer input to produce a tensor of outputs. When using this layer as the first layer in a model, provide the keyword argument input_shape (tuple of integers, does not include the sample axis), e.g. ```input_shape=(128, 128, 3)```. \n\n\n#### Parameters:\n1. ```Number of filters```: The number of output filters in the convolution\n\n\n\n2. ```Kernel Size```: An integer or tuple/list of 2 integers, specifying the height and width of the 2D convolution window. Can be a single integer to specify the same value for all spatial dimensions.\n\n3. ```RELU```: RELU layer will apply an elementwise activation function, such as the max(0,x) thresholding at zero. This leaves the size of the volume unchanged.\n\n4. ```Stride```: the number of pixels shifts over the input matrix. When the stride is 1 then we move the filters to 1 pixel at a time. When the stride is 2 then we move the filters to 2 pixels at a time and so on.\n\n\nTo know about other parameters, go [here](http://man.hubwiz.com/docset/TensorFlow.docset/Contents/Resources/Documents/api_docs/python/tf/keras/layers/Conv2D.html)\n\n### Batch Normalizarion layer\nTraining deep neural networks with tens of layers is challenging as they can be sensitive to the initial random weights and configuration of the learning algorithm.\n\nOne possible reason for this difficulty is the distribution of the inputs to layers deep in the network may change after each mini-batch when the weights are updated. This can cause the learning algorithm to forever chase a moving target. This change in the distribution of inputs to layers in the network is referred to the technical name “internal covariate shift.”\n\nBatch normalization is a technique for training very deep neural networks that standardizes the inputs to a layer for each mini-batch. This has the effect of stabilizing the learning process and dramatically reducing the number of training epochs required to train deep networks.\n\n### Dropout layer\n\nThe Dropout layer randomly sets input units to 0 with a frequency of rate at each step during training time, which helps prevent overfitting. Inputs not set to 0 are scaled up by 1/(1 - rate) such that the sum over all inputs is unchanged.\n\nNote that the Dropout layer only applies when training is set to True such that no values are dropped during inference.\n\n### Flatten Layer\n<img src = \"https://static.wixstatic.com/media/5adb1b_ef9160a7c14b4b93978297508262dfeb~mv2.png/v1/fill/w_640,h_370,al_c,q_85,usm_0.66_1.00_0.01,enc_auto/5adb1b_ef9160a7c14b4b93978297508262dfeb~mv2.png\" width = \"40%\"/>\n\n### Dense Layer\nFlatten is the function that converts the pooled feature map to a single column that is passed to the fully connected layer. Dense adds the fully connected layer to the neural network.","metadata":{}},{"cell_type":"markdown","source":"# Modeling\n\n### Approach:\n* Experimenting by looping over different parameters\n* Explore and infer\n* Choose the best and keep building the final model\n\n\n### Parameter #1: Number of convoluton layers","metadata":{}},{"cell_type":"code","source":"num_cnn_layers = 3\n\n# initialize models as empty list\nmodel = history = [0] * num_cnn_layers\n\n\n# loop over to form models\nfor i in range(num_cnn_layers):\n    # Layer 1\n    model[i] = Sequential(\n        (\n            Conv2D(32, kernel_size=5, activation=\"relu\", input_shape=(28, 28, 1)),\n            MaxPool2D(),\n        )\n    )\n\n    # Layer 2\n    if i > 0:\n        model[i].add(Conv2D(64, kernel_size=4, activation=\"relu\"))\n        model[i].add(MaxPool2D())\n\n    # Layer 3\n    if i > 1:\n        model[i].add(Conv2D(128, kernel_size=2, activation=\"relu\"))\n        model[i].add(MaxPool2D())\n\n    model[i].add(Flatten())\n\n    # NN Layer 1\n    model[i].add(Dense(256, activation=\"relu\"))\n    # NN layer 2\n    model[i].add(Dense(10, activation=\"softmax\"))\n\n    model[i].compile(\n        optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n    )\n\n# Train model\n\nfor i in range(num_cnn_layers):\n    history[i] = model[i].fit(\n        x, y, epochs=25, shuffle=True, validation_split=0.2, verbose=0\n    )\n    print(\n        \"Layer=\",\n        i + 1,\n        \" Epochs=25\",\n        \" Train accuracy=\",\n        max(history[i].history[\"accuracy\"]),\n        \" Validation accuracy=\",\n        max(history[i].history[\"val_accuracy\"]),\n    )","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2023-02-25T16:55:40.189482Z","iopub.execute_input":"2023-02-25T16:55:40.189884Z","iopub.status.idle":"2023-02-25T17:12:15.944334Z","shell.execute_reply.started":"2023-02-25T16:55:40.189849Z","shell.execute_reply":"2023-02-25T17:12:15.943134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot\nnames = [\"#1\", \"#2\", \"#3\"]\nplt.figure(figsize=(15, 5))\nfor i in range(num_cnn_layers):\n    plt.plot(history[i].history[\"val_accuracy\"])\nplt.title(\" Number of convoluton layers\")\nplt.ylabel(\"accuracy\")\nplt.xlabel(\"epoch\")\nplt.legend(names, loc=\"upper left\")\naxes = plt.gca()\naxes.set_ylim([0.93, 1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T17:12:15.946387Z","iopub.execute_input":"2023-02-25T17:12:15.946755Z","iopub.status.idle":"2023-02-25T17:12:16.136493Z","shell.execute_reply.started":"2023-02-25T17:12:15.946723Z","shell.execute_reply":"2023-02-25T17:12:16.135277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference:\n\nFrom the above experiment, it seems that 3 pairs of convolution-subsambling is slightly better than 2 pairs. However for efficiency, the improvement doesn't warrant the additional computional cost, so let's use 2.","metadata":{}},{"cell_type":"markdown","source":"### Parameter #2: Number of filter layers\n\n* Number of Filters we will explore in layer 1, 2:\n    * (8, 16)\n    * (16, 32)\n    * (24, 48)\n    * (32, 64)\n    * (40, 80)\n    * (48, 96)\n\n\nNote that as we move to next CNN layer, we tend to increase the number of filters, a common practice is to double it. We do this so as to get finer and increased number of features from the image.","metadata":{}},{"cell_type":"code","source":"model = history = [0] * 6\nfor i in range(6):\n    model[i] = Sequential(\n        (\n            Conv2D(\n                (i + 1) * 8, kernel_size=5, activation=\"relu\", input_shape=(28, 28, 1)\n            ),\n            MaxPool2D(),\n            Conv2D((i + 1) * 16, kernel_size=5, activation=\"relu\"),\n            MaxPool2D(),\n            Flatten(),\n            Dense(256, activation=\"relu\"),\n            Dense(10, activation=\"softmax\"),\n        )\n    )\n    model[i].compile(\n        optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n    )\n\nfor i in range(6):\n    history[i] = model[i].fit(\n        x, y, epochs=25, shuffle=True, validation_split=0.2, verbose=0\n    )\n    print(\n        \"Filters=\",\n        (i + 1) * 8,\n        \" Epochs=25\",\n        \" Train accuracy=\",\n        max(history[i].history[\"accuracy\"]),\n        \" Validation accuracy=\",\n        max(history[i].history[\"val_accuracy\"]),\n    )","metadata":{"execution":{"iopub.status.busy":"2023-02-25T17:12:16.138135Z","iopub.execute_input":"2023-02-25T17:12:16.138583Z","iopub.status.idle":"2023-02-25T17:48:29.716982Z","shell.execute_reply.started":"2023-02-25T17:12:16.138541Z","shell.execute_reply":"2023-02-25T17:48:29.715731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT ACCURACIES\nnames = [\"(8, 16)\", \"(16, 32)\", \"(24, 48)\", \"(32, 64)\", \"(48, 96)\", \"(64, 128)\"]\nplt.figure(figsize=(15, 5))\nfor i in range(6):\n    plt.plot(history[i].history[\"val_accuracy\"])\nplt.title(\"Number of filter layers\")\nplt.ylabel(\"accuracy\")\nplt.xlabel(\"epoch\")\nplt.legend(names, loc=\"upper left\")\naxes = plt.gca()\naxes.set_ylim([0.97, 1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T17:51:51.649095Z","iopub.execute_input":"2023-02-25T17:51:51.649552Z","iopub.status.idle":"2023-02-25T17:51:51.888712Z","shell.execute_reply.started":"2023-02-25T17:51:51.649514Z","shell.execute_reply":"2023-02-25T17:51:51.887352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference\nIt appears that 32 iflter set in the first convolutional layer and 64 in the second layer is the most optimal to choose. Architectures with more maps only perform slightly better and are not worth the additonal computation cost.\n\n\n\n### Parameter #3: Dropout\n\nThis will avoid overfitting. We will be looking for a dropout that increases the validation accuracy, thus helping to generalize better.\n* Dropouts tested: 0.3, 0.4, 0.5, 0.6, 0.7\n    \n","metadata":{}},{"cell_type":"code","source":"model = history = [0] * 5\n\nfor i in range(5):\n    model[i] = Sequential(\n        (\n            Conv2D(32, kernel_size=5, activation=\"relu\", input_shape=(28, 28, 1)),\n            MaxPool2D(),\n            Dropout(i * 0.1),\n            Conv2D(64, kernel_size=5, activation=\"relu\"),\n            MaxPool2D(),\n            Dropout(i * 0.1),\n            Flatten(),\n            Dense(256, activation=\"relu\"),\n            Dropout(i * 0.1),\n            Dense(10, activation=\"softmax\"),\n        )\n    )\n    model[i].compile(\n        optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"]\n    )\n\nfor i in range(5):\n    history[i] = model[i].fit(\n        x, y, epochs=25, shuffle=True, validation_split=0.2, verbose=0\n    )\n    print(\n        \"Dropout =\",\n        i + 3,\n        \" Epochs=25\",\n        \" Train accuracy=\",\n        max(history[i].history[\"accuracy\"]),\n        \" Validation accuracy=\",\n        max(history[i].history[\"val_accuracy\"]),\n    )","metadata":{"execution":{"iopub.status.busy":"2023-02-25T17:56:19.154604Z","iopub.execute_input":"2023-02-25T17:56:19.155094Z","iopub.status.idle":"2023-02-25T18:30:28.697130Z","shell.execute_reply.started":"2023-02-25T17:56:19.155055Z","shell.execute_reply":"2023-02-25T18:30:28.695896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PLOT ACCURACIES\nnames = [\"#3\", \"#4\", \"#5\", \"#6\", \"#7\"]\nplt.figure(figsize=(15, 5))\nfor i in range(5):\n    plt.plot(history[i].history[\"val_accuracy\"])\nplt.title(\"Dropouts\")\nplt.ylabel(\"accuracy\")\nplt.xlabel(\"epoch\")\nplt.legend(names, loc=\"upper left\")\naxes = plt.gca()\naxes.set_ylim([0.975, 1])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T18:34:24.058829Z","iopub.execute_input":"2023-02-25T18:34:24.059644Z","iopub.status.idle":"2023-02-25T18:34:24.295099Z","shell.execute_reply.started":"2023-02-25T18:34:24.059593Z","shell.execute_reply":"2023-02-25T18:34:24.294260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inference\n50% dropout seems to work the best and optimal","metadata":{}},{"cell_type":"markdown","source":"# Conclusion\n\nTraining convolutional neural networks is quite a random process. This makes experiments difficult because each time you run the same experiment, you get different results. Thus,running models dozens of times and taking an average can help. The best yet CNN architecture from few layers of CNN for classifying MNIST handwritten digits is with 32 filters, kernel size 5, and 2 layes of convolution with 50% dropout. \n\nFurther, **implementing deeper models** does improves accuracy. And adding batch normalizaiton and max pool improves the accuracy. ","metadata":{}},{"cell_type":"markdown","source":"## A deep network\n\nSpecifications: \n* 7 layered Conv network\n* 32-32-32-64-64-64-128 filters each\n* 3, 4, 5 kernel size\n* activation - relu\n* optimizer - adam\n* loss - categorical cross entropy\n","metadata":{}},{"cell_type":"code","source":"def build_model(input_shape=(28, 28, 1)):\n    model = Sequential(\n        (\n            # convolutional layer with rectified linear unit activation\n            # 32 convolution filters used each of size 3x3\n            Conv2D(32, kernel_size=3, activation=\"relu\", input_shape=input_shape),\n            BatchNormalization(),\n            Conv2D(32, kernel_size=3, activation=\"relu\"),\n            BatchNormalization(),\n            Conv2D(32, kernel_size=5, strides=2, padding=\"same\", activation=\"relu\"),\n            BatchNormalization(),\n            # randomly turn neurons on and off to improve convergence and avoid overfitting\n            Dropout(0.4),\n            Conv2D(64, kernel_size=3, activation=\"relu\"),\n            BatchNormalization(),\n            Conv2D(64, kernel_size=3, activation=\"relu\"),\n            BatchNormalization(),\n            Conv2D(64, kernel_size=5, strides=2, padding=\"same\", activation=\"relu\"),\n            BatchNormalization(),\n            Dropout(0.4),\n            Conv2D(128, kernel_size=4, activation=\"relu\"),\n            BatchNormalization(),\n            # flatten since too many dimensions, we only want a classification output\n            Flatten(),\n            Dropout(0.4),\n            # fully connected to get all relevant data\n            Dense(10, activation=\"softmax\"),\n        )\n    )\n    return model\n\n\ndef compile_model(model, optimizer=\"adam\", loss=\"categorical_crossentropy\"):\n    model.compile(optimizer=optimizer, loss=loss, metrics=[\"accuracy\"])\n\n\ndef train_model(model, train, test, epochs, split):\n    history = model.fit(\n        train, test, shuffle=True, epochs=epochs, validation_split=split, verbose=0\n    )\n    return history","metadata":{"execution":{"iopub.status.busy":"2023-02-25T19:31:24.738519Z","iopub.execute_input":"2023-02-25T19:31:24.739036Z","iopub.status.idle":"2023-02-25T19:31:24.753289Z","shell.execute_reply.started":"2023-02-25T19:31:24.738995Z","shell.execute_reply":"2023-02-25T19:31:24.751906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training the model using the above function built to build, compile and train the model\ncnn_model = build_model((28, 28, 1))\ncompile_model(cnn_model, \"adam\", \"categorical_crossentropy\")\n\n# train the model for as many epochs as you want but I found training it above 80 will not help us and eventually increase overfitting.\nmodel_history = train_model(cnn_model, x, y, 50, 0.2)","metadata":{"execution":{"iopub.status.busy":"2023-02-25T19:31:31.768081Z","iopub.execute_input":"2023-02-25T19:31:31.768571Z","iopub.status.idle":"2023-02-25T20:12:51.748715Z","shell.execute_reply.started":"2023-02-25T19:31:31.768526Z","shell.execute_reply":"2023-02-25T20:12:51.747615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_history.history.keys()","metadata":{"execution":{"iopub.status.busy":"2023-02-25T20:12:51.751038Z","iopub.execute_input":"2023-02-25T20:12:51.751407Z","iopub.status.idle":"2023-02-25T20:12:51.759932Z","shell.execute_reply.started":"2023-02-25T20:12:51.751374Z","shell.execute_reply":"2023-02-25T20:12:51.758549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_performance(metric, validations_metric):\n    plt.plot(\n        model_history.history[metric], label=str(\"Training: \" + metric), color=\"#FF9CEE\"\n    )\n    plt.plot(\n        model_history.history[validations_metric],\n        label=str(\"Validation: \" + metric),\n        color=\"#6EB5FF\",\n    )\n    plt.legend()\n\n\nplot_model_performance(\"accuracy\", \"val_accuracy\")","metadata":{"execution":{"iopub.status.busy":"2023-02-25T11:42:54.650629Z","iopub.status.idle":"2023-02-25T11:42:54.651014Z","shell.execute_reply.started":"2023-02-25T11:42:54.650820Z","shell.execute_reply":"2023-02-25T11:42:54.650838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_performance(\"loss\", \"val_loss\")","metadata":{"execution":{"iopub.status.busy":"2023-02-25T11:42:54.652188Z","iopub.status.idle":"2023-02-25T11:42:54.652584Z","shell.execute_reply.started":"2023-02-25T11:42:54.652378Z","shell.execute_reply":"2023-02-25T11:42:54.652395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### With pooling","metadata":{}},{"cell_type":"code","source":"def build_model(input_shape=(28, 28, 1)):\n    # Three steps to create a CNN\n    # 1. Convolution\n    # 2. Activation\n    # 3. Pooling\n    # Repeat Steps 1,2,3 for adding more hidden layers\n\n    # 4. After that make a fully connected network\n    # This fully connected network gives ability to the CNN\n    # to classify the samples\n\n    model = Sequential(\n        (\n            # 32 filters\n            Conv2D(32, (3, 3), activation=\"relu\", input_shape=(28, 28, 1)),\n            BatchNormalization(axis=-1),\n            Conv2D(32, (3, 3), activation=\"relu\"),\n            BatchNormalization(axis=-1),\n            # Pooling layer\n            MaxPooling2D(pool_size=(2, 2)),\n            # increase the number of filters\n            Conv2D(64, (3, 3), activation=\"relu\"),\n            BatchNormalization(axis=-1),\n            Conv2D(64, (3, 3), activation=\"relu\"),\n            BatchNormalization(axis=-1),\n            MaxPooling2D(pool_size=(2, 2)),\n            Flatten(),\n            # 1st hidden layer in fully connected layer\n            # 512 nodes\n            Dense(512),\n            Dropout(0.2),\n            Dense(10, activation=\"softmax\"),\n        )\n    )\n\n    return model\n\n\ndef compile_model(model, optimizer=\"adam\", loss=\"categorical_crossentropy\"):\n    model.compile(optimizer=optimizer, loss=loss, metrics=[\"accuracy\"])\n\n\ndef train_model(model, train, test, epochs, split):\n    history = model.fit(\n        train, test, shuffle=True, epochs=epochs, validation_split=split, verbose=0\n    )\n    return history","metadata":{"execution":{"iopub.status.busy":"2023-02-25T11:42:54.654008Z","iopub.status.idle":"2023-02-25T11:42:54.654424Z","shell.execute_reply.started":"2023-02-25T11:42:54.654216Z","shell.execute_reply":"2023-02-25T11:42:54.654235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model = build_model((28, 28, 1))\n\n# add optimizer(gradient boost)\n# add criterian for loss\ncompile_model(cnn_model, \"adam\", \"categorical_crossentropy\")\n\n# train the model with parameters:\n# model: declared above\n# x - features: pixels for each image in 28x28x1 format\n# y - target: 1 hot coded target array of dimension 1x10 for each image\n# number of epochs\n# validation split\n# returns\nmodel_history = train_model(cnn_model, x, y, 50, 0.2)","metadata":{"execution":{"iopub.status.busy":"2023-02-25T11:42:54.655695Z","iopub.status.idle":"2023-02-25T11:42:54.656256Z","shell.execute_reply.started":"2023-02-25T11:42:54.655882Z","shell.execute_reply":"2023-02-25T11:42:54.655900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_model_performance(metric, validations_metric):\n    plt.plot(\n        model_history.history[metric], label=str(\"Training: \" + metric), color=\"#FF9CEE\"\n    )\n    plt.plot(\n        model_history.history[validations_metric],\n        label=str(\"Validation: \" + metric),\n        color=\"#6EB5FF\",\n    )\n    plt.legend()\n\n\nplot_model_performance(\"accuracy\", \"val_accuracy\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_model_performance(\"loss\", \"val_loss\")","metadata":{},"execution_count":null,"outputs":[]}]}