{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport imgaug.augmenters as iaa\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6dd10fe04982fb59ad177489270f3176b8f6b516"},"cell_type":"markdown","source":"# Read input"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d21a42cd02213e89bfba8df18f83b29e79b3e53c"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d5e3791e80e77dd2c9cae19cbc0f18180b99092"},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a10ed6a36ffdee102a876d41322c6564e101aa75"},"cell_type":"markdown","source":"# Seperate input and output features"},{"metadata":{"trusted":true,"_uuid":"a29912f9fb589a262b2af87f472e4639f6d61ce2"},"cell_type":"code","source":"X = df.iloc[:, 1:]\nY = df.iloc[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4f33b55b490a10dd6d9f51d662f4da61f51e08f"},"cell_type":"code","source":"X.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36b944e1d89eadc5a005ee74199cc1cfc68b9e7d"},"cell_type":"code","source":"Y.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8349edb45af0c902b799a7cb90efc510d0867ddd"},"cell_type":"markdown","source":"Normalize X. Since we are delaing with pixels and they are going to be between 0 to 255, lets normalize the input features by dividing by 255"},{"metadata":{"trusted":true,"_uuid":"c5b11b89b0d6ce9ee80521072d06535e848cea12"},"cell_type":"code","source":"X = np.array(X)\nY = np.array(Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"840b875772753cc91485eae774e42f434606ab98"},"cell_type":"code","source":"X = X/255.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6a618ddf081c06f8eb325b8000f1a6268d2b3cb0"},"cell_type":"markdown","source":"# Plot images"},{"metadata":{"trusted":true,"_uuid":"e900a2f26db5cb0ef221adfb1c164484dc4df52c"},"cell_type":"code","source":"def plot_images(X, Y, shape):\n    for i in range(20):\n        plt.subplot(5, 4, i+1)\n        plt.tight_layout()\n        plt.imshow(X[i].reshape((shape,shape)), cmap='gray')\n        plt.title('Digit:{}'.format(Y[i]))\n        plt.xticks([])\n        plt.yticks([])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56a8d0cc927bd1a3b339591ac12c5d97528112f0"},"cell_type":"code","source":"plot_images(X[:20], Y[:20], 28)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17bb775a7fb214924486ccb51e7e65e43e6c17ce"},"cell_type":"markdown","source":"# Util class \nWill contain the activation functions along with their derivatives, cost functions and accuracy"},{"metadata":{"trusted":true,"_uuid":"0a9667d05e9f6e262671c698867d985fd8c4fd49"},"cell_type":"code","source":"class Util:\n    def __init__(self):\n        self.name = \"Util\"\n        \n    def sigmoid(self, z):\n        return 1 / (1 + np.exp(-z))\n    \n    def sigmoid_prime(self, z):\n        return -np.exp(-z)/(1 + np.exp(-z))**2\n    \n    def softmax(self, z):\n        expA = np.exp(z)\n        return expA/expA.sum(axis=1, keepdims=True)\n        \n    def softmax_prime(self, z):\n        s = self.softmax(z)\n        return s*(1-s)\n    \n    def tanh(self, z):\n        return np.tanh(z)\n    \n    def tanh_prime(self, z):\n        return 1 - np.tanh(z)**2\n    \n    def relu(self, z):\n        return np.where(z>0, z, 0)\n    \n    def relu_prime(self, z):\n        return np.where(z>0, 1, 0)\n    \n    def sigmoid_cost_function(self, P, T):\n        m = P.shape[0]\n        return -np.sum(T*np.log(P) + (1-T)*np.log(1-P))/m\n    \n    def softmax_cost_function(self, P, T, parameters, lambd):\n        m = P.shape[0]\n        sum_weights = 0\n        L = len(parameters)//2\n        for l in range(L):\n            sum_weights += np.sum(parameters[\"W\" + str(l + 1)])\n        J = (lambd / (2*m)) * (sum_weights**2)\n        J += -np.sum(T*np.log(P))/m\n        return J\n    \n    def accuracy(self, P, Y):\n        Yhat = np.argmax(P, axis=1)\n        return np.mean(Yhat==Y)\n    \n    def yEncode(self, Y):\n        T = np.zeros((len(Y), len(set(Y))))\n        T[np.arange(Y.size), Y] = 1\n        return T","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7db1c668d79754d416fb6c94e7c9911860d59308"},"cell_type":"markdown","source":"# Augment Images using ImgAug \n\nPerforms the following image Augmentation with ImgAug:\n* Flip\n* Scale\n* Translate\n* Rotate\n* Shear\n* Composite"},{"metadata":{"trusted":true,"_uuid":"6bb84cd6933b907ba00fa881c6f2601ed8a22a70"},"cell_type":"code","source":"class ImageAugmenter:\n    def __init__(self):\n        self.name = 'ImageAugmenter'\n        \n    def reshape_images(self, img_arr, shape):\n        return img_arr.reshape(shape)\n    \n    def transform_images(self, seq, img_arr, shape):\n        X_img = self.reshape_images(img_arr, (img_arr.shape[0], shape, shape))\n        X_aug = seq.augment_images(X_img)\n        X_aug = self.reshape_images(X_aug, (img_arr.shape[0], shape*shape))\n        return X_aug\n        \n    def fliplr(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Fliplr(1)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def flipud(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Flipud(1)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def scale(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                scale={\"x\":(0.5, 1.5), \"y\":(0.5, 1.5)}\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def translate(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                translate_percent={\"x\":(-0.2, 0.2), \"y\":(-0.2, 0.2)}\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def rotate(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                rotate=(-45, 45)\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def shear(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                shear=(-10, 10)\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def compose(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n            scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n            translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n            rotate=(-25, 25),\n            shear=(-8, 8)\n            ),\n            iaa.Pepper(1e-5)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def augment(self, count, X, Y):\n        X_out = np.copy(X)\n        for i in range(count):\n            X_scale = self.scale(X, 28)\n            X_rotate = self.rotate(X, 28)\n            X_trans = self.translate(X, 28)\n            X_shear = self.shear(X, 28)\n            X_compose = self.compose(X, 28)\n            X_out = np.concatenate((X_out, X_scale), axis = 0)\n            X_out = np.concatenate((X_out, X_rotate), axis = 0)\n            X_out = np.concatenate((X_out, X_trans), axis = 0)\n            X_out = np.concatenate((X_out, X_shear), axis = 0)\n            X_out = np.concatenate((X_out, X_compose), axis = 0)\n        Y = np.repeat(Y, (count*5)+1, axis = 0)\n        return X_out, Y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3c2e353e2f25e0aba8a3923d87910a67f5d5d2b"},"cell_type":"markdown","source":"# Core:\n\n1. Initialise Parameters\n2. Linear Forward\n3. Activation Forward\n4. Feed Forward\n5. Linear Backward\n6. Activation Backward\n7. Back Propagation"},{"metadata":{"trusted":true,"_uuid":"442390e1c8fd24b9d0c43ae60894886d8afb74b0"},"cell_type":"code","source":"class DNN_Core:\n    def __init__(self, layer_dims):\n        self.layer_dims = layer_dims\n        \n    def initialise_parameters(self):\n        parameters = {}\n        for l in range(1, len(self.layer_dims)):\n            parameters[\"W\" + str(l)] = np.random.randn(self.layer_dims[l-1], self.layer_dims[l])*0.01\n            parameters[\"b\" + str(l)] = np.random.randn(self.layer_dims[l], 1)*0.01\n        return parameters\n        \n    def linear_forward(self, A, W, b):\n        z = np.dot(A, W) + b.T\n        return z, (A, W, b)\n    \n    def activation_forward(self, A, W, b, activation):\n        util = Util()\n        z, linear_cache = self.linear_forward(A, W, b)\n        if activation == 'sigmoid':\n            A = util.sigmoid(z)\n        elif activation == 'softmax':\n            A = util.softmax(z)\n        elif activation == 'tanh':\n            A = util.tanh(z)\n        elif activation == 'relu':\n            A = util.relu(z)\n        activation_cache = z\n        return A, (linear_cache, activation_cache)\n    \n    def feed_forward(self, X, parameters):\n        caches = []\n        A = X\n        L = len(parameters)//2\n        for l in range(1, L):\n            A_prev = A\n            A, cache = self.activation_forward(A_prev, parameters[\"W\"+str(l)], parameters[\"b\"+str(l)], 'relu')\n            caches.append(cache)\n        P, cache = self.activation_forward(A, parameters['W'+str(L)], parameters['b'+str(L)], 'softmax')\n        caches.append(cache)\n        return P, caches\n    \n    def linear_backward(self, dZ, linear_cache):\n        A_prev, W, b = linear_cache\n        m = A_prev.shape[0]\n        dW = np.dot(A_prev.T, dZ)/m\n        db = np.sum(dZ, axis=0, keepdims=True)/m\n        dA_prev = np.dot(dZ, W.T)/m\n        return dA_prev, dW, db\n    \n    def activation_backward(self, dA, cache, activation):\n        util = Util()\n        linear_cache, activation_cache = cache\n        if activation == 'sigmoid':\n            dZ = dA * util.sigmoid_prime(activation_cache)\n        elif activation == 'softmax':\n            dZ = dA\n        elif activation == 'tanh':\n            dZ = dA * util.tanh_prime(activation_cache)\n        elif activation == 'relu':\n            dZ = dA * util.relu_prime(activation_cache)\n        dA_prev, dW, db = self.linear_backward(dZ, linear_cache)\n        return dA_prev, dW, db.T\n    \n    def back_propogation(self, P, T, caches):\n        gradients = {}\n        dA = P-T\n        L = len(caches)\n        dA_prev, dW, db = self.activation_backward(dA, caches[L-1], 'softmax')\n        gradients['dW'+str(L)] = dW\n        gradients['db'+str(L)] = db\n        gradients['dA'+str(L-1)] = dA_prev\n        for l in reversed(range(L-1)):\n            dA_prev, dW, db = self.activation_backward(dA_prev, caches[l], 'relu')\n            gradients['dW'+str(l+1)] = dW\n            gradients['db'+str(l+1)] = db\n            gradients['dA'+str(l)] = dA_prev\n        return gradients\n    \n    def update_patameters(self, parameters, gradients, alpha, lambd, m):\n        L = len(parameters)//2\n        for l in range(L):\n            parameters['W'+str(l+1)] -= alpha*gradients['dW'+str(l+1)] \n            parameters['W'+str(l+1)] -= lambd*parameters['W'+str(l+1)]\n            parameters['b'+str(l+1)] -= alpha*gradients['db'+str(l+1)]\n        return parameters","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f8dfc8ec17ff048e8ce721fd366016f82dd5671"},"cell_type":"markdown","source":"# Stochastic Gradient Descent \n** Batch size of 100 **\n\n** Shuffled dataset for each epoch **"},{"metadata":{"trusted":true,"_uuid":"89cf4f6a7ed2999ebf8d195bc3f62f54afbd6976"},"cell_type":"code","source":"class SGD:\n    def __init__(self, layer_dims):\n        self.layer_dims = layer_dims\n        \n    def SGD(self, X, X_val, Y, Y_val, epochs, alpha, lambd, batch_size):\n        J_history = []\n        J_val_history = []\n        acc_history = []\n        acc_val_history = []\n        core = DNN_Core(self.layer_dims)\n        util = Util()\n        augmenter = ImageAugmenter()\n        T = util.yEncode(Y)\n        T_val = util.yEncode(Y_val)\n        parameters = core.initialise_parameters()\n        for i in range(epochs):\n            start = 0\n            s = np.arange(X.shape[0])\n            np.random.shuffle(s)\n            X = X[s]\n            T = T[s]\n            Y = Y[s]\n            while start < X.shape[0]:\n                end = start + batch_size\n                if end > X.shape[0]:\n                    end = X.shape[0]\n                x = X[start:end, :]\n                t = T[start:end, :]\n                #x, t = augmenter.augment(x, t)\n                P, caches = core.feed_forward(x, parameters)\n                gradients = core.back_propogation(P, t, caches)\n                parameters = core.update_patameters(parameters, gradients, alpha, lambd, t.shape[0])\n                start = end\n            # Validate at the end of 10 epochs\n            if i%20 == 0:\n                P, _ = core.feed_forward(X, parameters)\n                P_val, _ = core.feed_forward(X_val, parameters)\n                J = util.softmax_cost_function(P, T, parameters, lambd)\n                J_val = util.softmax_cost_function(P_val, T_val, parameters, lambd)\n                J_history.append(J)\n                J_val_history.append(J_val)\n                acc = util.accuracy(P, Y)\n                acc_val = util.accuracy(P_val, Y_val)\n                acc_history.append(acc)\n                acc_val_history.append(acc_val)\n                print('Epoch: ' + str(i), 'Cost: {0:.2f} Valid_Cost: {1:.2f} Accuracy: {2:.7f} Valid_Accuracy: {3:.7f}'.format(J, J_val, acc, acc_val))\n        return P, parameters, J_history, J_val_history, acc_history, acc_val_history","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"125ca7bf8d5a0a1855c5ce43abfecf754e854d61"},"cell_type":"markdown","source":"# Fit the model"},{"metadata":{"trusted":true,"_uuid":"25016a22268ac3c9e9d2ee1c10394569b707bb8f"},"cell_type":"code","source":"from sklearn.model_selection import  train_test_split\nlayer_dims = [X.shape[1], 30, 20, 10]\nX_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.05, random_state=28) \nsgd = SGD(layer_dims)\nP, parameters, J_history, J_val_history, acc_history, acc_val_history = sgd.SGD(X_train, X_val, Y_train, Y_val, 500, 1e1, 1e-4, 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b50a06a3f3a73f765adc2c481e84a747c5ca2cd0"},"cell_type":"markdown","source":"# Plot the cost and accuracy"},{"metadata":{"trusted":true,"_uuid":"ff310b9c5d7573c36d0c12339c2644f029b9963e"},"cell_type":"code","source":"plt.plot(J_history, color='blue')\nplt.plot(J_val_history, color='red')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6539c3274689317f286c5f4520219a83375f4913"},"cell_type":"code","source":"plt.plot(acc_history, color='blue')\nplt.plot(acc_val_history, color='red')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a6f81ad32cfb83a9cb12769e4664a1db9d569fee"},"cell_type":"markdown","source":"# Validation dataset prediction"},{"metadata":{"trusted":true,"_uuid":"be5bb47a90a6c34a42a944b9730ca09a37856b75"},"cell_type":"code","source":"core = DNN_Core(layer_dims)\nutil = Util()\nP_test, _ = core.feed_forward(X_val, parameters)\nprint(util.accuracy(P_test, Y_val))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"06924145c0ba4b71cb30293020cec2c7adf30353"},"cell_type":"markdown","source":"# Create output"},{"metadata":{"trusted":true,"_uuid":"7c8159c1bb36a702e69692eea0f3c8b621d8e924"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13deba1b0447de676f0648e8b8637c49ad420878"},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"259f2f36ac037124a24eb00504252423b6d8465a"},"cell_type":"code","source":"df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f68617cb52ef13655f9a29b98475ebc7855ae0f"},"cell_type":"code","source":"X_out = np.array(df_test)\nX_out = X_out/255.0\nP_out, _ = core.feed_forward(X_out, parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9a23adec0cae4b1892054b51c8e6c9ee48fcb7c"},"cell_type":"code","source":"Y_out = np.argmax(P_out, axis=1)\nY_out[1:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4586c9c4183f2080ec0cd54b05b4877abb11c745"},"cell_type":"code","source":"df_submission = pd.read_csv('../input/sample_submission.csv')\ndf_submission['Label'] = Y_out\ndf_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f56f9857e591c58f88b68e5daa376309db2fb9d0"},"cell_type":"code","source":"df_submission.to_csv('output.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}