{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport imgaug.augmenters as iaa\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15a964862882ed0be86a19512ffdc4ece2d3dd3b"},"cell_type":"markdown","source":"# Load input data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e65305f8b2c7a320a8898601034c520841e21bbc"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea9caa82364099b76751edb0175580854d8f7135"},"cell_type":"code","source":"X_train = df_train.iloc[:, 1:]\nY_train = df_train.iloc[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c42ce4f3f3456a517742e64cf81d4101137a75"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"905ad01f0362853ed9c5e34de31b8fc359874264"},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5458393be93bffde30b22258c017f813bd33fad2"},"cell_type":"code","source":"Y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8045abdce3cf6acbdd167f9c9690fcb6b76882ce"},"cell_type":"code","source":"X_train = np.array(X_train)\nY_train = np.array(Y_train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3355b1a77bc1cedb11df7c272d8f3f498dca01c"},"cell_type":"markdown","source":"# Plot images"},{"metadata":{"trusted":true,"_uuid":"e6b04bafd3a4c4f6c84463979a7d7bc4a2afd84f"},"cell_type":"code","source":"def plot_digits(X, y, dim):\n    fig = plt.figure()\n    for i in range(9):\n        plt.subplot(3,3,i+1)\n        plt.tight_layout()\n        plt.imshow(X[i].reshape((dim, dim)), interpolation='none', cmap='gray')\n        plt.title(\"Digit: {}\".format(y[i]))\n        plt.xticks([])\n        plt.yticks([])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2def62f0daf76677e856337a3b998309b52ee6ae"},"cell_type":"code","source":"plot_digits(X_train, Y_train, 28)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bc76780df89b3ec3ca2a5995460c06a7f2340b9"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\nsns.countplot(Y_train)\nax.set_title(\"Distribution of Digits\", fontsize=12)\nax.set_xlabel(\"Digits\", fontsize=10)\nax.set_ylabel('Count', fontsize=10)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f82c1dc58550d78ec8118b8ef0301faf60ac9347"},"cell_type":"markdown","source":"We have almost equal distribution of digits."},{"metadata":{"_uuid":"21203cc359f2ec49b0344aedccb275ce9cd2d2f1"},"cell_type":"markdown","source":"# Normalize Data"},{"metadata":{"trusted":true,"_uuid":"c971568367aa2f3346e622761d4700d22b3f1daa"},"cell_type":"code","source":"X_train = X_train/255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c8413b1d7a814e46fc4b8e1f8534bf8a0ec0f5f"},"cell_type":"code","source":"plot_digits(X_train, Y_train, 28)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"feb15f9022265c4d66b9ec2a43abf45efac5b381"},"cell_type":"markdown","source":"# Train Test Split"},{"metadata":{"trusted":true,"_uuid":"620f043f695b13012ebe52306af70fdc8dbe3809"},"cell_type":"code","source":"X_dev, X_val, Y_dev, Y_val = train_test_split(X_train, Y_train, test_size=0.03, shuffle=True, \\\n                                              random_state=2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c400d253e990b1cc8a6c408e3bd3ab0eae9e6cf1"},"cell_type":"code","source":"#Encode Y\ndef yEncode(Y):\n    #T = np.zeros((Y.size, len(set(Y))))\n    #T[np.arange(Y.size), Y] = 1\n    T = pd.get_dummies(Y).values\n    return T\n\nT_dev = yEncode(Y_dev)\nT_val = yEncode(Y_val)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d9220855e324a2babdc6848d6d5cd7eaa01c958"},"cell_type":"markdown","source":"# Augmentation Implementation"},{"metadata":{"trusted":true,"_uuid":"f9f4a761607361d78f4e8423ca067feddc843960"},"cell_type":"code","source":"class ImageAugmenter:\n    def __init__(self):\n        self.name = 'ImageAugmenter'\n        \n    def reshape_images(self, img_arr, shape):\n        return img_arr.reshape(shape)\n    \n    def transform_images(self, seq, img_arr, shape):\n        X_img = self.reshape_images(img_arr, (img_arr.shape[0], shape, shape))\n        X_aug = seq.augment_images(X_img)\n        X_aug = self.reshape_images(X_aug, (img_arr.shape[0], shape*shape))\n        return X_aug\n        \n    def fliplr(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Fliplr(1)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def flipud(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Flipud(1)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def scale(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                scale={\"x\":(0.5, 1.5), \"y\":(0.5, 1.5)}\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def translate(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                translate_percent={\"x\":(-0.2, 0.2), \"y\":(-0.2, 0.2)}\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def rotate(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                rotate=(-45, 45)\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def shear(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n                shear=(-10, 10)\n            )\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def compose(self, X, shape):\n        seq = iaa.Sequential([\n            iaa.Affine(\n            scale={\"x\": (0.8, 1.2), \"y\": (0.8, 1.2)},\n            translate_percent={\"x\": (-0.2, 0.2), \"y\": (-0.2, 0.2)},\n            rotate=(-25, 25),\n            shear=(-8, 8)\n            ),\n            iaa.Pepper(1e-5)\n        ])\n        return self.transform_images(seq, X, shape)\n    \n    def augment(self, count, X, Y):\n        X_out = np.copy(X)\n        for i in range(count):\n            #X_scale = self.scale(X, 28)\n            #X_rotate = self.rotate(X, 28)\n            #X_trans = self.translate(X, 28)\n            #X_shear = self.shear(X, 28)\n            X_compose = self.compose(X, 28)\n            #X_out = np.concatenate((X_out, X_scale), axis = 0)\n            #X_out = np.concatenate((X_out, X_rotate), axis = 0)\n            #X_out = np.concatenate((X_out, X_trans), axis = 0)\n            #X_out = np.concatenate((X_out, X_shear), axis = 0)\n            X_out = np.concatenate((X_out, X_compose), axis = 0)\n        Y = np.repeat(Y, (count*1)+1, axis = 0)\n        return X_out, Y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79ea3f1848b88cf0071a6ab04f7b66e5fabd54b1"},"cell_type":"markdown","source":"# Placeholders for inputs and dropouts"},{"metadata":{"trusted":true,"_uuid":"e22db936af8f8b38892fd458ea3fe438c9eb7f3d"},"cell_type":"code","source":"X = tf.placeholder(tf.float32, shape=[None, 784], name='x')\ny = tf.placeholder(tf.float32, shape=[None, 10], name='labels')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61d5a5e86cef47be5347b511f5af55bd57f07116"},"cell_type":"code","source":"keep_prob_1 = tf.placeholder(tf.float32)\nkeep_prob_2 = tf.placeholder(tf.float32)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f611c794a2cf865a33dc9e9eff82a991984df9b5"},"cell_type":"markdown","source":"# Initialise Parameters"},{"metadata":{"trusted":true,"_uuid":"2bc8d52fcb8fc19defa5bd9abbb4f1c40a2b05d1"},"cell_type":"code","source":"def get_W(name, values):\n    W = tf.Variable(values, name=name)\n    return W","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44a38d7f2262e14b4c98b2f69600d26552f5de8b"},"cell_type":"code","source":"def get_b(name, values):\n    b = tf.Variable(values, name=name)\n    return b","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27a78fb7ffa17df05d4269ecfc1ec9990f4b8d5a"},"cell_type":"markdown","source":"# Graph Functions"},{"metadata":{"trusted":true,"_uuid":"ecfb673b666bf43059fa1ff3032de4707bb7bede"},"cell_type":"code","source":"def linear_forward(X, W, b):\n    z = tf.matmul(X, W) + b\n    return z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d51a8729ae1b1cba97480637910907d0b22b4bac"},"cell_type":"code","source":"def activation_forward(X, W, b, activation):\n    z = linear_forward(X, W, b)\n    if activation == 'sigmoid':\n        return tf.nn.sigmoid(z)\n    elif activation == 'softmax':\n        return tf.nn.softmax(z)\n    elif activation == 'tanh':\n        return tf.nn.tanh(z)\n    elif activation == 'relu':\n        return tf.nn.relu(z)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe7421c1cb9d5b26c5879934cdb333e6fe4dcc24"},"cell_type":"code","source":"def fc_layer(X, n_units, name, activation):\n    W = get_W(name, tf.truncated_normal([X.shape[1].value, n_units], stddev=0.1))\n    b = get_b(name, tf.zeros([1, n_units]))\n    return activation_forward(X, W, b, activation)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"195b30a067f255835d761156b765879dd449aab3"},"cell_type":"markdown","source":"# Network Architecture"},{"metadata":{"trusted":true,"_uuid":"d4ca9b80446619ca89ce21bb9a6e0bfb8166d79d"},"cell_type":"code","source":"dropout_x = tf.nn.dropout(X, keep_prob=keep_prob_1)\nfc1 = fc_layer(dropout_x, 324, 'fc1', 'relu')\ndropout_fc1 = tf.nn.dropout(fc1, keep_prob=keep_prob_2)\nfc2 = fc_layer(dropout_fc1, 100, 'fc2', 'relu')\ndropout_fc2 = tf.nn.dropout(fc2, keep_prob=keep_prob_2)\nfc3 = fc_layer(dropout_fc2, 100, 'fc3', 'relu')\ndropout_fc3 = tf.nn.dropout(fc3, keep_prob=keep_prob_2)\nfc4 = fc_layer(dropout_fc3, 100, 'fc4', 'relu')\ndropout_fc4 = tf.nn.dropout(fc4, keep_prob=keep_prob_2)\nfc5 = fc_layer(dropout_fc4, 225, 'fc5', 'relu')\ndropout_fc5 = tf.nn.dropout(fc5, keep_prob=keep_prob_2)\nout = fc_layer(dropout_fc5, 10, 'out', 'softmax')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3206cee25d6990d995fdd4c504365accaefda487"},"cell_type":"markdown","source":"# Hyperparameters"},{"metadata":{"trusted":true,"_uuid":"56e05e92c1d68be1508494bdd884ff1c8f9f535f"},"cell_type":"code","source":"epoch = 20\nlearning_rate = 5e-4\nbatch_size = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a01235c063b870a39052181162b8d2d8dbd01578"},"cell_type":"markdown","source":"# Cost, Accuracy and Optimizer"},{"metadata":{"trusted":true,"_uuid":"7d727e8e277110d7d64dca8ece7e78a5329d8c79"},"cell_type":"code","source":"loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=out))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75bd657804df7e77eb1676b41b6078bdeb19e77f"},"cell_type":"code","source":"print(learning_rate)\ntrain = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fb4908cf782e3b30fc6a0232f94f76a62e6f882"},"cell_type":"code","source":"equal_pred = tf.equal(tf.argmax(y,1), tf.argmax(out,1))\nacc = tf.reduce_mean(tf.cast(equal_pred, tf.float32))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"258db448468d0f8e2224d48c2b5e52cae3ff6192"},"cell_type":"markdown","source":"# Initialise all variables"},{"metadata":{"trusted":true,"_uuid":"a0addbe7a0a0fa33d9011c070781b71f41c06120"},"cell_type":"code","source":"init = tf.global_variables_initializer()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ad0a8818a6925cbaea57bf9f771d55b46d7ce15"},"cell_type":"markdown","source":"# Training the model"},{"metadata":{"trusted":true,"_uuid":"ded80280a690dd665b49086aced129f4bfdb9f20","scrolled":false},"cell_type":"code","source":"#Create session and initialise global variables\nsess = tf.Session()\nsess.run(init)\naugmenter = ImageAugmenter()\n\nfor i in range(epoch):\n    start_index = 0\n    s = np.arange(X_dev.shape[0])\n    np.random.shuffle(s)\n    X_dev = X_dev[s,:]\n    T_dev = T_dev[s]\n    while start_index < X_dev.shape[0]:\n        end_index = start_index + batch_size\n        if end_index > X_dev.shape[0]:\n            end_index = X_dev.shape[0]\n        x_dev = X_dev[start_index:end_index, :]\n        y_dev = T_dev[start_index:end_index]\n        #x_dev, y_dev = augmenter.augment(1, x_dev, y_dev)\n        #print(x_dev.shape, y_dev.shape)\n        cost, _, accuracy, pred, fc = sess.run([loss, train, acc, out, fc2], \\\n                                    feed_dict={X: x_dev, y:y_dev, keep_prob_1:1, keep_prob_2:1})\n        start_index = end_index\n    t_cost, t_acc = sess.run([loss, acc], \\\n                                       feed_dict={X:X_dev, y:T_dev, keep_prob_1:1, keep_prob_2:1})\n    v_cost, v_acc = sess.run([loss, acc], \\\n                                       feed_dict={X:X_val, y:T_val, keep_prob_1:1, keep_prob_2:1})\n    X_grad, fc1_grad, fc2_grad, fc3_grad, fc4_grad, fc5_grad = \\\n                    sess.run([X, fc1, fc2, fc3, fc4, fc5], \\\n                        feed_dict={X:X_dev, y:T_dev, keep_prob_1:1, keep_prob_2:1})\n    Y_dev = np.argmax(T_dev, axis=1)\n    print(\"Epoch:\", (i+1), \"cost =\", \"{:.5f}\".format(t_cost), \"acc =\", \"{:.5f}\".format(t_acc), \\\n             \"val_cost = {:.5f}\".format(v_cost), \"val_acc = {:.5f}\".format(v_acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e88dfa6edeb769f20092b1630b566a0e06d0103"},"cell_type":"code","source":"plt.imshow(X_dev[2].reshape((28,28))), np.argmax(T_dev[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9449c38a4103db9dc0eb31c552a572a4bde9b1e2"},"cell_type":"code","source":"X_test = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d1e55c65764f066f7ca2066a34ed3abcec1f8f9"},"cell_type":"code","source":"X_test = np.array(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fde58beed371807f699faf7e0c5254e9f5fc390"},"cell_type":"code","source":"out_test = sess.run([out], feed_dict={X:X_test, keep_prob_1:1, keep_prob_2:1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e60f031ec48ca16277152db2596ff850a59cf9aa"},"cell_type":"code","source":"P_test = np.argmax(out_test[0], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eec1c11407ee3ad9d56d048c0d5ff72f3ec2ffdc"},"cell_type":"code","source":"df_out = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03b158d706e231dc19ef123d652ca16b969e5187"},"cell_type":"code","source":"df_out['Label'] = P_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38509ec901489d8af1fed471955150cf5f0be616"},"cell_type":"code","source":"df_out.to_csv('out.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}