{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom sklearn.model_selection import train_test_split\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport matplotlib.ticker as mticker\n\nstyle.use('fivethirtyeight')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Try image augmentaion (rotation, transformations)\n#https://www.kaggle.com/dhayalkarsahilr/easy-image-augmentation-techniques-for-mnist\n\n# Load the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n\nprint (type(train.iloc[1,1]))\nprint (type(test.iloc[1,1]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22db6485256f34ab45ae6433144dbb966a1daa19","_cell_guid":"7aff1293-a84c-46fd-9047-3db6e1852546","trusted":true},"cell_type":"code","source":"label = train[\"label\"]\n# Convert to one_hot array\nLabels = np.zeros((label.size, label.max()+1))\nLabels[np.arange(label.size),label] = 1\n\n# Drop 'label' column\nFeatures = train.drop(labels = [\"label\"],axis = 1)\n\ntrainFeatures, testFeatures, trainLabels, testLabels = train_test_split(Features,Labels,test_size=0.2,random_state=0)\n#np.array(trainFeatures)\n#np.array(trainLabels)\n\n#trainLabels.head()\nprint (trainFeatures.shape)\nprint (trainLabels.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"482366949da90ca41c83a3a198aff459edfd64a3","_cell_guid":"94abb32d-d331-49e7-aa61-171f28f72c43","trusted":true},"cell_type":"code","source":"# Set some model parameters\nn_nodes = 2500\nn_classes = 10\n\nx = tf.placeholder('float', [None, 784])\ny = tf.placeholder('float', [None, 10])\nkeep_prob = tf.placeholder(tf.float32)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eb093a2b58f3dac09f51fe91c0ec6a90ccb290fa","_cell_guid":"27cbeaa6-a45d-416a-b58b-eb6d2d560232","trusted":true},"cell_type":"code","source":"# Define the neural network\ndef neural_network_model(data, keep_prob):\n    hidden_1_layer = {'weights':tf.Variable(tf.random_normal([784, n_nodes])),\n                      'biases':tf.Variable(tf.random_normal([n_nodes]))}\n\n    hidden_2_layer = {'weights':tf.Variable(tf.random_normal([n_nodes, n_nodes])),\n                      'biases':tf.Variable(tf.random_normal([n_nodes]))}\n\n    hidden_3_layer = {'weights':tf.Variable(tf.random_normal([n_nodes, n_nodes])),\n                      'biases':tf.Variable(tf.random_normal([n_nodes]))}\n\n    hidden_4_layer = {'weights':tf.Variable(tf.random_normal([n_nodes, n_nodes])),\n                      'biases':tf.Variable(tf.random_normal([n_nodes]))}\n\n    hidden_5_layer = {'weights':tf.Variable(tf.random_normal([n_nodes, n_nodes])),\n                      'biases':tf.Variable(tf.random_normal([n_nodes]))}\n\n    output_layer = {'weights':tf.Variable(tf.random_normal([n_nodes, n_classes])),\n                    'biases':tf.Variable(tf.random_normal([n_classes])),}\n\n\n    l1 = tf.add(tf.matmul(data,hidden_1_layer['weights']), hidden_1_layer['biases'])\n    l1 = tf.nn.relu(l1)\n    l1 = tf.nn.dropout(l1, keep_prob)\n\n    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weights']), hidden_2_layer['biases'])\n    l2 = tf.nn.relu(l2)\n    l2 = tf.nn.dropout(l2, keep_prob)\n\n    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weights']), hidden_3_layer['biases'])\n    l3 = tf.nn.relu(l3)\n    l3 = tf.nn.dropout(l3, keep_prob)\n\n    #l4 = tf.add(tf.matmul(l3,hidden_4_layer['weights']), hidden_4_layer['biases'])\n    #l4 = tf.nn.relu(l4)\n    #l4 = tf.nn.dropout(l4, keep_prob)\n\n    #l5 = tf.add(tf.matmul(l4,hidden_5_layer['weights']), hidden_5_layer['biases'])\n    #l5 = tf.nn.relu(l5)\n    #l5 = tf.nn.dropout(l5, keep_prob)\n\n    # Add dropout layer\n    # dropout = tf.matmult(l5,0.6)\n    \n    output = tf.matmul(l3,output_layer['weights']) + output_layer['biases']\n    # output = tf.matmul(dropout,output_layer['weights']) + output_layer['biases']\n    \n    return output\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4da852dd07c963a9e428e0be1174ab0d60495a8d","_cell_guid":"522fd63a-8c70-450a-a71f-dcc421636e99","trusted":true},"cell_type":"code","source":"# Run the neural network session.\ndef train_neural_network(x):\n    \n    prediction = neural_network_model(x, 0.5)\n    cost = tf.reduce_mean( tf.nn.\n                softmax_cross_entropy_with_logits_v2(logits=prediction, labels=y) )\n    optimizer = tf.train.AdamOptimizer().minimize(cost)\n    # adjust learning rate\n    # data augmentation\n    # 60 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.927381\n    \n    num_epochs = 50\n    # 10 epochs, 1 layers, 500 nodes/layer, Adam Optimizer: 0.12238095\n    # 30 epochs, 1 layers, 500 nodes/layer, Adam Optimizer: 0.27964285\n    # 50 epochs, 1 layers, 500 nodes/layer, Adam Optimizer: 0.3727381\n    \n    # 10 epochs, 3 layers, 500 nodes/layer, Adam Optimizer: 0.12464286\n    # 30 epochs, 3 layers, 500 nodes/layer, Adam Optimizer: 0.18059523\n    # 50 epochs, 3 layers, 500 nodes/layer, Adam Optimizer: 0.2325\n    \n    # 10 epochs, 3 layers, 1000 nodes/layer, Adam Optimizer: 0.1370238\n    # 30 epochs, 3 layers, 1000 nodes/layer, Adam Optimizer: 0.27595237\n    # 50 epochs, 3 layers, 1000 nodes/layer, Adam Optimizer: 0.43785715\n    \n    # 10 epochs, 3 layers, 1500 nodes/layer, Adam Optimizer: 0.1575\n    # 30 epochs, 3 layers, 1500 nodes/layer, Adam Optimizer: 0.37238094\n    # 50 epochs, 3 layers, 1500 nodes/layer, Adam Optimizer: 0.5221428\n    \n    # 10 epochs, 5 layers, 500 nodes/layer, Adam Optimizer: 0.10666667\n    # 30 epochs, 5 layers, 500 nodes/layer, Adam Optimizer: 0.10607143\n    # 50 epochs, 5 layers, 500 nodes/layer, Adam Optimizer: 0.112261906\n    \n    # 10 epochs, 5 layers, 1000 nodes/layer, Adam Optimizer: 0.103452384\n    # 30 epochs, 5 layers, 1000 nodes/layer, Adam Optimizer: 0.12630953\n    # 50 epochs, 5 layers, 1000 nodes/layer, Adam Optimizer: 0.13666667\n    \n    # 10 epochs, 5 layers, 1500 nodes/layer, Adam Optimizer: 0.10916667\n    # 30 epochs, 5 layers, 1500 nodes/layer, Adam Optimizer: 0.15214285\n    # 50 epochs, 5 layers, 1500 nodes/layer, Adam Optimizer: 0.17392857\n    \n    # 10 epochs, 3 layers, 2000 nodes/layer, Adam Optimizer: 0.21035714\n    # 10 epochs, 5 layers, 2000 nodes/layer, Adam Optimizer: 0.12309524\n    \n    # 50 epochs, 3 layers, 2000 nodes/layer, Adam Optimizer: 0.60464287\n    # 50 epochs, 3 layers, 2500 nodes/layer, Adam Optimizer: 0.64488095\n    \n    # 30 epochs, 5 layers, 2000 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 5 layers, 3000 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 7 layers, 2000 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 7 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 7 layers, 3000 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 9 layers, 2000 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 9 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 30 epochs, 9 layers, 3000 nodes/layer, Adam Optimizer: 0.\n    \n    # 30 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 40 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 50 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 60 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 70 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 80 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 90 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    # 100 epochs, 5 layers, 2500 nodes/layer, Adam Optimizer: 0.\n    \n    with tf.Session() as sess:\n        \n        tf.initialize_all_variables().run()\n        sess.run(tf.global_variables_initializer())\n\n        for epoch in range(num_epochs):\n            epoch_loss = 0\n            _, c = sess.run([optimizer, cost], \n                    feed_dict={x: trainFeatures, y: trainLabels, keep_prob: 0.5})\n            epoch_loss += c\n\n            print('Epoch', epoch, 'completed out of',num_epochs,'loss:',epoch_loss)\n            \n        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))\n\n        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))\n        print('Accuracy:',accuracy.eval({x:testFeatures, y:testLabels}))\n        \n        \n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f29e384d60857548086eab5987a54b33812d55c7"},"cell_type":"code","source":"print (x.shape)\nprint (trainFeatures.shape)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d9215b8b-7d88-48bb-8906-9cce9ea357b9","_uuid":"a7663a8a5a49c13e34e89d9843f23c6f3d546512","scrolled":false,"trusted":true},"cell_type":"code","source":"# Train the neural network\ntrain_neural_network(x)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74d85f1c5f84287fe2fcd26bf329321cb1c616f7","_cell_guid":"9fa1f4cd-e187-4489-be4e-509574731f49","trusted":true},"cell_type":"code","source":"# Make some predictions\n# Convert test set to float\ntest = test.astype(np.float32, copy=False)\nprediction = neural_network_model(test,1.0)\n\nwith tf.Session() as sess:\n    sess.run(tf.global_variables_initializer())\n    \n    test_pred = sess.run(prediction, feed_dict={x:test, keep_prob: 1})\n    test_labels = np.argmax(test_pred, axis=1)\n\nprint (test_pred.shape)\nprint (test_labels.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b5bc8c3de35c1f2166655a9adb1f490ab25890b"},"cell_type":"code","source":"print (test_pred[0])\nprint (test_labels[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a7115f4f3acd37ca68bf4dabafdcd9f1db56311","_cell_guid":"877c7c8e-2511-4e95-a0ae-01bee4768393","trusted":true},"cell_type":"code","source":"# plot the first 100 test images and see how good we were.\nfig = plt.figure(figsize=(15,15))\nax = []\nsquare = 10\nfor i in range(square*square):\n    ax.append(plt.subplot2grid((square,square), (i%square,int(i/square)), rowspan=1, colspan=1))\n\n    pixels = test.iloc[i,:].values\n    pixels = pixels.reshape((28, 28))\n\n    ax[i].get_xaxis().set_visible(False)\n    ax[i].get_yaxis().set_visible(False)\n    ax[i].xaxis.set_major_locator(mticker.MaxNLocator(6))\n    ax[i].yaxis.set_major_locator(mticker.MaxNLocator(6))\n    ax[i].set_title('{label}'.format(label=test_labels[i]),fontdict={'fontsize':10})\n    ax[i].imshow(pixels, cmap='gray')\n\nplt.show()\n# Anywhere from 6-10% right. This is crap :(","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"595cb9e097fdc9067b201908b2272c2be4c427c2","_cell_guid":"cc8a42ac-59f7-49b8-b311-7daf4afaf586","trusted":true},"cell_type":"code","source":"# Histogram of predicted values\nplt.figure(figsize=(12,8))\nplt.hist(test_labels, histtype='bar', rwidth=0.8)\nplt.xlabel('Predicted Label')\nplt.ylabel('Count')\nplt.title('Frequency of predictions')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1a74394ffd3f699c1146ad8985a74b13b3e3dec7","_cell_guid":"3487b6d1-2637-404e-ba8f-d8aaa96ce208","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82bb96c065e10f33cbe5655a6092d7e3adeae6c9","_cell_guid":"f965b1d8-ec55-4e24-b2db-f49a049747de","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a46e5def0d3282fe8aea4a80cbf6975fd15645a","_cell_guid":"24aba6e8-8516-41ce-8413-227dabdabfdc","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}