{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport time\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\").values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b235a88106750a9addc3690bb0ee7491956c60da"},"cell_type":"code","source":"X = train[:, 1:]\nY = train[:, 0]\n\nimg_num = 6001\nplt.imshow(X=X[img_num].reshape(28, 28), cmap=plt.get_cmap('gray'))\nplt.title(\"Y[\" + str(img_num) + \"] = \" + str(Y[img_num]))\n\nprint(\"Y[\" + str(img_num) + \"] = \" + str(Y[img_num]))\nY_tensor = tf.transpose(tf.one_hot(indices=Y, depth=10, axis=0))\nwith tf.Session() as sess:\n    Y = sess.run(Y_tensor)\nprint(\"One hot encoded Y[\" + str(img_num) + \"] = \" + str(Y[img_num]))\nprint()\n\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42)\nX_train = X_train.T\nY_train = Y_train.T\nX_test = X_test.T\nY_test = Y_test.T\nprint(\"X_train shape: \" + str(X_train.shape))\nprint(\"Y_train shape: \" + str(Y_train.shape))\nprint(\"X_test shape: \" + str(X_test.shape))\nprint(\"Y_test shape: \" + str(Y_test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac9ec1d16531d32b1893d81659b1b523bd2ae054"},"cell_type":"code","source":"def create_placeholders(n_x, n_y):\n    X = tf.placeholder(dtype=tf.float32, shape=[n_x, None])\n    Y = tf.placeholder(dtype=tf.float32, shape=[n_y, None])\n    keep_prob = tf.placeholder(dtype=tf.float32)\n    return X, Y, keep_prob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b0c85e2c6a4255f42745c7faf5b2f0205bd7c5a"},"cell_type":"code","source":"def initialize_parameters(n_x, n_y, hidden_layer_dims):\n    weights = []\n    biases = []\n    prev_dim = n_x\n    layer_num = 1\n    for dim in hidden_layer_dims:\n        weight = tf.get_variable(name=\"W\" + str(layer_num), shape=[dim, prev_dim], initializer=tf.contrib.layers.xavier_initializer(seed=1),\n                                 dtype=tf.float32, regularizer=tf.contrib.layers.l2_regularizer(scale=0.0))\n        bias = tf.get_variable(name=\"b\" + str(layer_num), shape=[dim, 1], initializer=tf.zeros_initializer(), dtype=tf.float32)\n        weights.append(weight)\n        biases.append(bias)\n        prev_dim = dim\n        layer_num+=1\n    weight = tf.get_variable(name=\"W\" + str(layer_num), shape=[n_y, prev_dim], initializer=tf.contrib.layers.xavier_initializer(seed=1),\n                             dtype=tf.float32, regularizer=tf.contrib.layers.l2_regularizer(scale=0.0))\n    bias = tf.get_variable(name=\"b\" + str(layer_num), shape=[n_y, 1], initializer=tf.zeros_initializer(), dtype=tf.float32)\n    weights.append(weight)\n    biases.append(bias)\n    return {\"weights\": weights, \"biases\": biases}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35bf747f16dac0ae4d1c3aea474d8595ca824d31"},"cell_type":"code","source":"tf.reset_default_graph()\nwith tf.Session() as sess:\n    parameters = initialize_parameters(784, 10, [512])\n    print(\"Weights: \" + str(parameters[\"weights\"]))\n    print(\"Biases: \" + str(parameters[\"biases\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d078b953ae36bd3f93c95d82cee9f1c9bb961387"},"cell_type":"code","source":"def forward_propagation(parameters, X, keep_prob):\n    weights = parameters[\"weights\"]\n    biases = parameters[\"biases\"]\n    A_prev = X\n    for (W,b) in zip(weights, biases):\n        Z = tf.matmul(W, A_prev) + b\n        A = tf.nn.relu(Z)\n        A_dropout = tf.nn.dropout(A, keep_prob)\n        A_prev = A_dropout\n    return Z","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25075c06d3772e0b9b5d4fdf2e53320066a2d9f1"},"cell_type":"code","source":"tf.reset_default_graph()\n\nwith tf.Session() as sess:\n    X, Y, keep_prob = create_placeholders(784, 10)\n    parameters = initialize_parameters(784, 10, [512])\n    Z = forward_propagation(parameters, X, keep_prob)\n    print(\"Z4 = \" + str(Z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"253e6bcdb608a006324312c67031319f03d55132"},"cell_type":"code","source":"def compute_cost(Z, Y):\n    logits = tf.transpose(Z)\n    labels = tf.transpose(Y)\n    return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits, labels=labels)) + tf.losses.get_regularization_loss()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"352d0d0b8a89660bb95c3d9e03897e50edfebb71"},"cell_type":"code","source":"import os\nimport sys\n\nSTDOUT = 1\nSTDERR Attempting to use uninitialized value= 2\n\nclass FDRedirector(object):\n    \"\"\" Class to redirect output (stdout or stderr) at the OS level using\n        file descriptors.\n    \"\"\" \n\n    def __init__(self, fd=STDOUT):\n        \"\"\" fd is the file descriptor of the outpout you want to capture.\n            It can be STDOUT or STERR.\n        \"\"\"\n        self.fd = fd\n        self.started = False\n        self.piper = None\n        self.pipew = None\n\n    def start(self):\n        \"\"\" Setup the redirection.\n        \"\"\"\n        if not self.started:\n            self.oldhandle = os.dup(self.fd)\n            self.piper, self.pipew = os.pipe()\n            os.dup2(self.pipew, self.fd)\n            os.close(self.pipew)\n\n            self.started = True\n\n    def flush(self):\n        \"\"\" Flush the captured output, similar to the flush method of any\n        stream.\n        \"\"\"\n        if self.fd == STDOUT:\n            sys.stdout.flush()\n        elif self.fd == STDERR:\n            sys.stderr.flush()\n\n    def stop(self):\n        \"\"\" Unset the redirection and return the captured output. \n        \"\"\"\n        if self.started:\n            self.flush()\n            os.dup2(self.oldhandle, self.fd)\n            os.close(self.oldhandle)\n            f = os.fdopen(self.piper, 'r')\n            output = f.read()\n            f.close()\n\n            self.started = False\n            return output\n        else:\n            return ''\n\n    def getvalue(self):\n        \"\"\" Return the output captured since the last getvalue, or the\n        start of the redirection.\n        \"\"\"\n        output = self.stop()\n        self.start()\n        return output\n    \n    #redirect=FDRedirector(STDERR)\n    #redirect.start();\n    #print(\"Result\")\n    #print(redirect.stop())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"190eb1cd732172776c8cad9afaecc0c2dc622cca"},"cell_type":"code","source":"def model(X_train, Y_train, X_test, Y_test, hidden_layer_dims, learning_rate = 0.0005, keep_prob=1.0, epoch_count = 50):\n    \n    tf.reset_default_graph()\n    \n    n_x, m = X_train.shape\n    n_y, _ = Y_train.shape\n\n    X, Y, dropout_keep_prob = create_placeholders(n_x, n_y)\n    parameters = initialize_parameters(n_x, n_y, hidden_layer_dims)\n    ZL = forward_propagation(parameters, X, dropout_keep_prob)\n    cost = compute_cost(ZL, Y)\n    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n    \n    init = tf.global_variables_initializer()\n    \n    with tf.Session() as sess:\n        sess.run(init)\n        for epoch in range(epoch_count):\n            _, epoch_cost = sess.run([optimizer, cost], feed_dict={X: X_train, Y: Y_train, dropout_keep_prob: keep_prob})\n            if (epoch+1) % 50 == 0:\n                print(\"Cost at Epoch \" + str(epoch) + \":\" + str(epoch_cost))\n\n\n        parameters = sess.run(parameters)\n        print (\"Parameters have been trained!\")\n        \n        correct_prediction = tf.equal(tf.argmax(ZL), tf.argmax(Y))\n\n        accuracy = tf.reduce_mean(tf.cast(correct_prediction, dtype=tf.float32))\n\n        print (\"Train Accuracy:\", accuracy.eval({X: X_train, Y: Y_train, dropout_keep_prob: 1.0}))\n        print (\"Test Accuracy:\", accuracy.eval({X: X_test, Y: Y_test, dropout_keep_prob: 1.0}))\n        \n        return parameters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c04e81f9af1425c10dc434f4ba8c2510254d113b","scrolled":true},"cell_type":"code","source":"start_time = time.time()\nparameters = model(X_train, Y_train, X_test, Y_test, [1000, 750, 500, 200, 100], keep_prob=0.9, epoch_count=1000)\nduration = time.time() - start_time\nprint(\"Training time: \" + str(duration) + \"s\")\n#print(parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0ba29e21e2a7c997ba06961a6d5025d3c5247bc"},"cell_type":"code","source":"def predict(X, parameters):\n    X_test = tf.placeholder(shape=[X.shape[0], None], dtype=tf.float32)\n    ZL = forward_propagation(parameters, X_test, 1.0)\n    prediction = tf.argmax(ZL, axis=0)\n    \n    with tf.Session() as sess:\n        p = sess.run(prediction, feed_dict={X_test: X})\n        \n    return p","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c53f934890fb9312cab3554153ddb65d05565f90"},"cell_type":"code","source":"X_pred = pd.read_csv(\"../input/test.csv\").values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43582bc8a224bac4979893bbd9cca74b2882b4de"},"cell_type":"code","source":"def convert_to_tensor(params):\n    weights = params[\"weights\"]\n    biases = params[\"biases\"]\n    weight_tensors = []\n    bias_tensors = []\n    for (w,b) in zip(weights, biases):\n        weight_tensors.append(tf.convert_to_tensor(w))\n        bias_tensors.append(tf.convert_to_tensor(b))\n    return {\"weights\": weight_tensors, \"biases\": bias_tensors}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95e9571c2ee911e233dcd34067b89cb4f1f4d51e"},"cell_type":"code","source":"img_num = -3\nplt.imshow(X=X_pred[img_num].reshape(28, 28), cmap=plt.get_cmap('gray'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb4231c693ef79fd9b2ccccbc235d13bb129a3d0"},"cell_type":"code","source":"parameters = convert_to_tensor(parameters)\nresults = predict(X_pred.T, parameters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f97349cee796195b5e6e6e0f207209c38780a505"},"cell_type":"code","source":"df = pd.DataFrame(results)\ndf.index.name='ImageId'\ndf.index+=1\ndf.columns=['Label']\ndf.to_csv('results2.csv', header=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67342648fd24b92effa410cc808f9256819571f3"},"cell_type":"markdown","source":"750,750|1000|Dropout(0.5)<br>\nCost at Epoch 999:0.30263132<br>\nTrain Accuracy: 0.99756616<br>\nTest Accuracy: 0.97261906<br>\nTraining time: 725.108108997345s<br>\n\n750,750|1000|Dropout(0.8)<br>\nCost at Epoch 999:0.011308524<br>\nTrain Accuracy: 1.0<br>\nTest Accuracy: 0.9766667<br>\nTraining time: 647.1035282611847s<br>\n\n750,750|1000|Dropout(0.9)<br>\nCost at Epoch 999:0.005368023<br>\nTrain Accuracy: 1.0<br>\nTest Accuracy: 0.9728571<br>\nTraining time: 643.4405686855316s<br>\n\n500,500|1500|Dropout(0.8)<br>\nCost at Epoch 1499:0.01156287<br>\nTrain Accuracy: 1.0<br>\nTest Accuracy: 0.9754762<br>\nTraining time: 974.1590616703033s<br>\n\n1000,750,500,200|1000|Dropout(0.9)<br>\nCost at Epoch 999:0.0043582385<br>\nTrain Accuracy: 1.0<br>\nTest Accuracy: 0.975<br>\nTraining time: 728.7301154136658s<br>\n<b>Submission Accuracy: 0.97771</b><br>"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}