{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nfrom tensorflow.python.client import device_lib\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.model_selection import train_test_split # sklearn, train_test_split\nimport matplotlib.pyplot as plt # matplotlib, plt\nfrom keras.utils.np_utils import to_categorical # categorical - one-hot-encoding\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n# CNN libraries\nfrom sklearn.metrics import confusion_matrix\nimport itertools\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\nimport math\nimport h5py\nimport scipy\nfrom PIL import Image\nfrom scipy import ndimage\nfrom tensorflow.python.framework import ops\nimport tensorflow.compat.v1 as tf\ntf.disable_v2_behavior()\n\n%matplotlib inline\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def separate_labels():\n    return [train[\"label\"], train.drop(labels = [\"label\"],axis=1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train, x_train = separate_labels()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_train.head(5))\nprint(y_train.head(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_figure():\n    for index in range(0, 9):\n        fig = x_train.iloc[index].to_numpy()\n        fig = fig.reshape((28,28))\n        plt.subplot(340 + (index + 1))\n        plt.imshow(fig)\n        plt.title(y_train[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_figure()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.to_numpy()\ntest = test.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Normalizing\nx_train = x_train/255\ntest = test/255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#x_train.shape\n#y_train.shape\ntest.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.reshape(-1,28,28,1)\nprint(x_train.shape)\ntest = test.reshape(-1,28,28,1)\nprint(test.shape)\nprint(y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = tf.keras.utils.to_categorical(y_train,10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Implementing forward prop by hand, for reference, we won't be using this","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_single_step(slice,W,b):\n    mul = slice * W\n    return (np.sum(mul) + b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_forward(A_prev,W,b,hyperpms):\n    # Retrieve dimensions from A_prev's shape (≈1 line)  \n    (m, n_H_prev, n_W_prev, n_C_prev) = A_prev.shape[0],A_prev.shape[1],A_prev.shape[2],A_prev.shape[3]\n    \n    # Retrieve dimensions from W's shape (≈1 line)\n    (f, f, n_C_prev, n_C) = W.shape[0],W.shape[1],W.shape[2],W.shape[3]\n    \n    stride = hyperpms['stride']\n    \n    n_H = int((n_H_prev - f)/stride + 1)\n    n_W = int((n_W_prev - f)/stride + 1)\n    \n    z = np.zeros((m,n_H,n_W,n_C))\n    \n    for i in range(m):\n        a_prev = A_prev[i]\n        for h in range(n_H):\n            vert_start = h*stride\n            vert_end = h*stride + f\n            for w in range(n_W):\n                hor_start = w*stride\n                hor_end = w*stride + f\n                for c in range(n_C):\n                    a_slice_prev = a_prev[vert_start:vert_end,hor_start:hor_end,:]\n                    weights = W[:,:,:,c]\n                    biases = b[:,:,:,c]\n                    z[i,h,w,c] = conv_single_step(a_slice_prev,weights,biases)\n                    \n    cache = (A_prev, W, b, hyperpms)\n    return z,cache\n                ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pool_forward(A_prev,hyperpms,mode='max'):\n    \n    (m,n_H_prev,n_W_prev,n_C_prev) = A_prev.shape\n    \n    f = hyperpms[\"f\"]\n    stride = hyperpms[\"stride\"]\n    \n    n_H = int((n_H_prev - f)/stride) + 1\n    n_W = int((n_W_prev - f)/stride) + 1\n    n_C = n_C_prev\n    \n    A = np.zeros((m,n_H,n_W,n_C))\n    \n    for i in range(m):\n        for h in range(n_H):\n            vert_start = h*stride\n            vert_end = h*stride + f\n            for w in range(n_W):\n                hor_start = w*stride\n                hor_end = w*stride + f\n                for c in range(n_C):\n                    a_slice_prev = A_prev[i,vert_start:vert_end,hor_start:hor_end,c]\n                    if mode == \"max\":\n                        A[i, h, w, c] = np.max(a_prev_slice)\n                    elif mode == \"average\":\n                        A[i, h, w, c] = np.mean(a_prev_slice)\n    cache = (A_prev,hyperpms)\n    return A,cache","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Tensorflow implementation from this point onwards","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_placeholders(n_H0,n_W0,n_C0,n_y):\n    X =  tf.compat.v1.placeholder('float',[None, n_H0, n_W0, n_C0])\n    Y =  tf.compat.v1.placeholder('float',[None, n_y])\n    \n    return X,Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def initialize_parameters():\n    W1 = tf.compat.v1.get_variable(\"W1\", [4,4,1,8], initializer=tf.glorot_uniform_initializer())\n    W2 = tf.compat.v1.get_variable(\"W2\", [2,2,1,16], initializer=tf.glorot_uniform_initializer())\n    \n    parameters = {\"W1\":W1, \"W2\":W2}\n    \n    return parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Forward prop using tensorflow this point onwards\n#CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED\ndef forward_prop(X,parameters):\n    \n    W1 = parameters[\"W1\"]\n    W2 = parameters[\"W2\"]\n    \n    z1 = tf.nn.conv2d(X,W1,strides=[1,1,1,1],padding = 'SAME')\n    a1 = tf.nn.relu(z1)\n    p1 = tf.nn.max_pool(a1, ksize=[1,4,4,1],strides=[1,4,4,1],padding = 'SAME')\n    \n    z2 = tf.nn.conv2d(p1,W2,strides=[1,1,1,1],padding=\"SAME\")\n    a2 = tf.nn.relu(z2)\n    p2 = tf.nn.max_pool(a2,ksize=[1,2,2,1],strides=[1,1,1,1],padding=\"SAME\")\n    \n    f =  tf.keras.layers.Flatten()(p2)\n    z3 = tf.keras.layers.Dense(10,activation=None)(f)\n    \n    return z3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_cost(z3,Y):\n    \n    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = z3,labels = Y))\n    \n    return cost","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def random_mini_batches(X, Y, mini_batch_size = 3000):\n    m = X.shape[0]                  # number of training examples\n    mini_batches = []\n        \n    # Step 1: Shuffle (X, Y)\n    permutation = list(np.random.permutation(m))\n    shuffled_X = X[permutation,:,:,:]\n    shuffled_Y = Y[permutation,:]#.reshape((Y.shape[0],m))\n    \n    # Step 2: Partition (shuffled_X, shuffled_Y). Minus the end case.\n    num_complete_minibatches = math.floor(m/mini_batch_size) # number of mini batches of size mini_batch_size in your partitionning\n    for k in range(0, num_complete_minibatches):\n        mini_batch_X = shuffled_X[k * mini_batch_size: (k+1)*mini_batch_size,:,:,:]\n        mini_batch_Y = shuffled_Y[k * mini_batch_size: (k+1)*mini_batch_size,:]\n        mini_batch = (mini_batch_X, mini_batch_Y)\n        mini_batches.append(mini_batch)\n    \n    # Handling the end case (last mini-batch < mini_batch_size)\n    if m % mini_batch_size != 0:\n        mini_batch_X = shuffled_X[num_complete_minibatches * mini_batch_size:,:,:,:]\n        mini_batch_Y = shuffled_Y[num_complete_minibatches * mini_batch_size:,:]\n        mini_batch = (mini_batch_X, mini_batch_Y)\n        mini_batches.append(mini_batch)\n    return mini_batches","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Model\n#create placeholders\n#initialize parameters\n#forward propagate\n#compute the cost\n#create an optimizer\ndef model(x_train,y_train,test,learning_rate=0.009,no_of_epochs=40,minibatch_size=3000,print_cost=True):\n    \n    ops.reset_default_graph()   \n    (m,n_H0,n_W0,n_C0) = x_train.shape\n    n_y = y_train.shape[1]\n    X,Y = create_placeholders(n_H0,n_W0,n_C0,n_y)\n    \n    costs=[]\n    \n    parameters = initialize_parameters()\n    \n    z3 = forward_prop(X,parameters)\n    \n    cost = compute_cost(z3,Y)\n    \n    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n    \n    init = tf.global_variables_initializer()\n    \n    with tf.compat.v1.Session() as sess:\n        \n        sess.run(init)\n        \n        for epoch in range(no_of_epochs):\n            \n            minibatch_cost = 0\n            num_minibatches = int(m/minibatch_size)\n            minibatches = random_mini_batches(x_train, y_train, minibatch_size)\n            \n            for minibatch in minibatches:\n                (minibatch_x,minibatch_y) = minibatch\n                _ , temp_cost = sess.run(fetches=[optimizer,cost],feed_dict={X:minibatch_x,Y:minibatch_y}) #this is why we had put 'None' in placholders\n                minibatch_cost += temp_cost/num_minibatches\n                    \n            if print_cost==True:\n                print(\"Cost after epoch %i: %f\" % (epoch, minibatch_cost))\n            costs.append(minibatch_cost)\n        \n        plt.plot(np.squeeze(costs))\n        plt.ylabel('cost')\n        plt.xlabel('iterations (per tens)')\n        plt.title(\"Learning rate =\" + str(learning_rate))\n        plt.show()\n\n        # Calculate the correct predictions\n        predict_op = tf.argmax(z3, 1)\n        correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))\n        \n        # Calculate accuracy on the test set\n        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n        print(accuracy)\n        train_accuracy = accuracy.eval({X: x_train, Y: y_train})\n        print(\"Train Accuracy:\", train_accuracy)\n        a = sess.run([tf.argmax(z3,1)], feed_dict={X: test})\n        a = np.transpose(a)\n        print(a)\n        sample_submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\n        sample_submission['Label'] = a\n        sample_submission.head()\n        # saving the file\n        sample_submission.to_csv('submission.csv', index=False)\n                \n        return train_accuracy, parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"_, parameters = model(x_train, y_train, test)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}