{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport numpy as np\nfrom sklearn.metrics import confusion_matrix\nimport time\nfrom datetime import timedelta\nimport math","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.__version__","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.datasets import mnist\n(x_train, y_train), (x_test, y_test) = mnist.load_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Size of:\")\nprint(\" - Training-set:\\t\\t{}\".format(len(x_train)))\nprint(\" - Test-set:\\t\\t\\t{}\".format(len(x_test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape, y_train.shape, x_test.shape, y_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function for plotting the images"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_images(images, cls_true, cls_pred=None):\n    assert len(images) == len(cls_true) == 9\n    \n    # Create figure with 3x3 sub-plots.\n    fig, axes = plt.subplots(3, 3)\n    fig.subplots_adjust(hspace=0.3, wspace=0.3)\n\n    for i, ax in enumerate(axes.flat):\n        # Plot image.\n        ax.imshow(images[i].reshape(img_shape), cmap='binary')\n\n        # Show true and predicted classes.\n        if cls_pred is None:\n            xlabel = \"True: {0}\".format(cls_true[i])\n        else:\n            xlabel = \"True: {0}, Pred: {1}\".format(cls_true[i], cls_pred[i])\n\n        # Show the classes as the label on the x-axis.\n        ax.set_xlabel(xlabel)\n        \n        # Remove ticks from the plot.\n        ax.set_xticks([])\n        ax.set_yticks([])\n    \n    # Ensure the plot is shown correctly with multiple plots\n    # in a single Notebook cell.\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Dimensions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# We know that MNIST images are 28 pixels by 28 pixels\nimg_size = 28\n\n# Images are stored in one-dimensional arrays of this length - number of numbers in the matrix of size 28 x 28\nimg_size_flat = img_size*img_size\n\n# Height and width of each image in Tuple\nimg_shape = (img_size, img_size)\n\n# Number of color channels for the image: 1 channel for gray-scale\nnum_channels = 1\n\n# Number of classes, one class for each of 10 digits\nnum_classes = 10","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the first images from the train set.\nimages = x_train[0:9]\n\n# Get the true classes for those images.\ncls_true = y_train[0:9]\n\n# Plot the images and labels using our helper-function above.\nplot_images(images=images, cls_true=cls_true)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparameters of the Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convolutional layer 1\nfilter1_size = 5  #Convolution filters are 5 x 5 pixels\nnum_filters1 = 16 #There are 16 of these filters\n\n# Convolutional layer 2\nfilter2_size = 5 #Convolution filters are 5 x 5 pixels\nnum_filters2 = 32 #There are 32 of these filters\n\n# Pooling\nwindow_size = 2 #Pooling window 2x2\nwindow_stride = 2 #Move by 2 strides\n\n# Fully-connected layer\nfc_size=128     # Number of nodes in the fully-connected layer\n\n# Convolution stride\nconv_stride=1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Transform the input data into the required shape"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare the input data\nx = tf.constant(x_train, tf.float32)\ny = tf.constant(y_train, tf.int64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape as required for ConvNet\nx_image = tf.reshape(x, [-1, img_size, img_size, num_channels])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_image.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Parameters for Convolutional Layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define shape for conv1 weights\nconv1_weight_shape = [filter1_size, filter1_size, num_channels, num_filters1]\n# Define conv1 weights\nconv1_weights = tf.Variable(tf.random.normal(shape=conv1_weight_shape, stddev=0.05))\n\n# Define shape for conv2 weights\nconv2_weight_shape = [filter2_size, filter2_size, num_filters1, num_filters2]\n# Define conv2 weights\nconv2_weights = tf.Variable(tf.random.normal(shape=conv2_weight_shape, stddev=0.05))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function for Convolutional Layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_layer(inputs, weights, num_filters, img_size, conv_stride, pooling_window_size, pooling_window_stride):\n    # Define shape for bias\n    bias_shape = [1, img_size, img_size, num_filters]\n    # Define bias\n    bias = tf.Variable(tf.ones(shape=bias_shape))\n    # Define convolutional layer\n    conv = tf.nn.conv2d(input=inputs, filters=weights, strides=conv_stride, padding='SAME')\n    # Add bias\n    conv+=bias\n    # Apply RELU activation function\n    conv = tf.nn.relu(conv)\n    # Apply Max Pooling\n    conv = tf.nn.max_pool(input=conv, ksize=pooling_window_size, strides=pooling_window_stride, padding='SAME')\n    return conv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate 1st convolutional layer\nconv1 = conv_layer(x_image, conv1_weights, num_filters1, img_size, conv_stride, window_size, window_stride)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calculate 2nd convolutional layer\nconv2 = conv_layer(conv1, conv2_weights, num_filters2, conv1.shape[1], conv_stride, window_size, window_stride)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function for flattening a layer"},{"metadata":{"trusted":true},"cell_type":"code","source":"def flatten_layer(layer):\n    # Get the shape of the layer\n    layer_shape = layer.get_shape()\n    # Calculate the number of features which is: img_height * img_width * num_channels\n    num_features = layer_shape[1:4].num_elements()\n    # Reshape the layer to [num_images, num_features].\n    # Note that we just set the size of the second dimension\n    # to num_features and the size of the first dimension to -1\n    # which means the size in that dimension is calculated\n    # so the total size of the tensor is unchanged from the reshaping.\n    flat_layer = tf.reshape(layer, [-1, num_features])\n    return flat_layer, num_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Flatten the output of 2nd convolutional layer \nflat_layer, num_features = flatten_layer(conv2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Define Parameters for the Fully Connected and Output Layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the weights for fully connected layer\nweight_fc = tf.Variable(tf.random.normal(shape=[num_features, fc_size]))\n# Initialize the bias for fully connected layer\nbias_fc = tf.Variable(tf.ones([fc_size]))\n\n# Define weights for the output layer\nweight_out = tf.Variable(tf.random.normal(shape=[fc_size, num_classes]))\n# Initialize the layer bias\nbias_out = tf.Variable(tf.ones([num_classes]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Construct the Forward Feeding Fully Connected Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the architecture of the model\ndef nn_forward(features, fc_weights, fc_bias, w_out, b_out):\n    # Fully connected layer\n    fc_product = tf.matmul(features, fc_weights)\n    # Apply activation function to the fully connected layer\n    fully_connected = tf.keras.activations.relu(fc_product+fc_bias)\n    \n    # Output layer\n    output = tf.matmul(fully_connected, w_out)\n    prediction = tf.keras.activations.softmax(output + b_out)\n    return prediction","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper function for the loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def nn_loss_function(features, fc_weights, fc_bias, w_out, b_out, actual):\n    prediction = nn_forward(features, fc_weights, fc_bias, w_out, b_out)\n    # y_pred_cls = tf.argmax(prediction, axis=1)\n    loss = tf.keras.losses.categorical_crossentropy(tf.one_hot(actual,depth=10), prediction)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model by learning the optimal weights\n\nopt = tf.keras.optimizers.Adam(0.5) # 1.Initialize optimizer with learning rate of 0.5\n\nfor j in range(1000): # 2. Search for the optimal values for the weights\n    opt.minimize(lambda: nn_loss_function(flat_layer, weight_fc, bias_fc, weight_out, bias_out, y), var_list=[weight_fc, bias_fc, weight_out, bias_out]) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Make prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = nn_forward(flat_layer, weight_fc, bias_fc, weight_out, bias_out)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Accuracy\nsum((tf.argmax(prediction, axis=1)==y).numpy())/len(y.numpy())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}