{"cells":[{"metadata":{"_cell_guid":"73f2e49e-67b8-41e5-880f-a6800c398187","_uuid":"373f1724c8fb586c0a35ce2569373b1e2779c2f9"},"cell_type":"markdown","source":"# Tensorflow tutorial on MNIST"},{"metadata":{"_cell_guid":"fae555b5-3fa8-4b4a-bb49-4c1ddd35ffe0","_uuid":"faab87250004262b73dcb1e5a1e623ecc892654b","colab_type":"text","id":"W_Hghyj7h2Zz"},"cell_type":"markdown","source":"### Resources\n\n\n\n*   Tutorial: https://www.tensorflow.org/versions/r1.1/get_started/mnist/beginners\n*   Code: https://github.com/tensorflow/tensorflow/blob/r1.1/tensorflow/examples/tutorials/mnist/mnist_softmax.py\n\nPosts to check later:\n*   Really cool post about **dim reduction**: http://colah.github.io/posts/2014-10-Visualizing-MNIST/\n*   Visual interpretation of NN: [http://colah.github.io/posts/2014-03-NN-Manifolds-Topology/](http://colah.github.io/posts/2014-03-NN-Manifolds-Topology/)\n\n"},{"metadata":{"_cell_guid":"0bd40799-03d6-45a6-8210-2932e0be9774","_uuid":"55a6e69cda8477895a83415664dadc1d364c737c","colab":{"autoexec":{"startup":false,"wait_interval":0},"base_uri":"https://localhost:8080/","height":122,"output_extras":[{"item_id":2}]},"colab_type":"code","collapsed":true,"executionInfo":{"elapsed":966,"status":"ok","timestamp":1520188759988,"user":{"displayName":"Alexandre Gomes","photoUrl":"//lh4.googleusercontent.com/-0SJODzHVIXQ/AAAAAAAAAAI/AAAAAAAAUeU/Cs1pVksWH3I/s50-c-k-no/photo.jpg","userId":"108678232453929155174"},"user_tz":0},"id":"9X2lJKJhSchd","outputId":"b64849a8-05dc-4b95-f4d9-1729c25ba8b7","trusted":true},"cell_type":"code","source":"from matplotlib import pyplot as plt \n%matplotlib notebook\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import OneHotEncoder\nfrom time import sleep as pause","execution_count":171,"outputs":[]},{"metadata":{"_cell_guid":"17e4c88b-0dab-466f-8c23-84609b7f3b6a","_uuid":"d912192a0d4bdd95746eabc1ce5d7d22943aef2e"},"cell_type":"markdown","source":"#### Build dataset structures as in the tutorial from the CSVs"},{"metadata":{"_cell_guid":"f2aa74e8-f55b-45ce-9ea4-1e962adbd312","_uuid":"83f5ec46931fd569347ba7befad186252ec5e444","collapsed":true,"trusted":true},"cell_type":"code","source":"train_raw_data = pd.read_csv('../input/train.csv').sample(100)\ntest_raw_data = pd.read_csv('../input/train.csv').sample(100)\n\n# create tensors\ntrain_labels, train_images = train_raw_data.pop(\"label\").values, train_raw_data.values\ntest_labels, test_images = test_raw_data.pop(\"label\").values, test_raw_data.values\n\n# apply one hot encoding to the labels\nOHEnconder = OneHotEncoder()\ntrain_labels = OHEnconder.fit_transform(train_labels.reshape(-1,1)).toarray()\ntest_labels = OHEnconder.fit_transform(test_labels.reshape(-1,1)).toarray()","execution_count":172,"outputs":[]},{"metadata":{"_cell_guid":"c266cb50-64f6-4734-99f4-52fa2ad921b8","_uuid":"9a55921d3c6869e23439b9fcf9781cc14c7ad09f"},"cell_type":"markdown","source":"### Build and run Tensorflow session"},{"metadata":{"_cell_guid":"bd05e695-a77e-44ee-82c9-c0267c96c432","_uuid":"74d51be704d3fac1843e10dd2cf5efd8f49d3599","colab":{"autoexec":{"startup":false,"wait_interval":0}},"colab_type":"code","collapsed":true,"id":"R02E6_CzufVH","trusted":true},"cell_type":"code","source":"def normalize(X):#linear rescaling to the range [-1,1]\n    rg = (X.max()-X.min())\n    return np.zeros(X.shape) if rg==0 else (X-X.min())/rg\n\n\n# initialize input placeholders and model variables\nx = tf.placeholder(tf.float32, [None,784])\ny_ = tf.placeholder(tf.float32, [None,10])\n\nW = tf.Variable(tf.zeros([784, 10])) # no hidden layers: input layer fully connected to the output layer\nb = tf.Variable(tf.zeros([10]))    \n\n# specify model output\ny = tf.matmul(x, W) + b\n\n# define loss function. Calculated from the cross entropy between the estimated output and the true label\n# loss_func = tf.reduce_mean( - tf.reduce_sum(y_ * tf.log(tf.softmax(y)),reduction_indices = [1]))# explicit implementation of cross entropy\nloss_func = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_,logits=y)) #nuerically stable implementation from tf\nloss=[]\n\n# define what to do in each training iteration\ntrain_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss_func)\n\n#Start a tensorflow session and initialize variables\nsess = tf.InteractiveSession()\ninit = tf.global_variables_initializer()\nsess.run(init)\n\n# Define accuracy\ncalc_accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(y,1), tf.argmax(y_,1)),tf.float32))","execution_count":173,"outputs":[]},{"metadata":{"_cell_guid":"51f5a8d4-4dc8-499c-9f24-30084ac35dba","_uuid":"104655c82d963d2048faeecc0e4b8418275c0b62","colab":{"autoexec":{"startup":false,"wait_interval":0},"base_uri":"https://localhost:8080/","height":51,"output_extras":[{"item_id":1}]},"colab_type":"code","executionInfo":{"elapsed":512,"status":"ok","timestamp":1520192904983,"user":{"displayName":"Alexandre Gomes","photoUrl":"//lh4.googleusercontent.com/-0SJODzHVIXQ/AAAAAAAAAAI/AAAAAAAAUeU/Cs1pVksWH3I/s50-c-k-no/photo.jpg","userId":"108678232453929155174"},"user_tz":0},"id":"nPNU-LDlgAt7","outputId":"1b210c70-5063-4e91-f7a2-eb02808d067d","trusted":true},"cell_type":"code","source":"# evaluate model accuracy before training. All weights and biases set to 0 so it should classify everythin as a 0\nprint(\"Percentage of 0's in the test set: {:.3f}\".format(test_labels.sum(axis=0)[0]/test_labels.sum()))\n\nacc =  sess.run(calc_accuracy, feed_dict={x: test_images,y_: test_labels})\nprint(\"Model accuracy: {:.3f}\".format(acc))\n\nplt.ion()\nfig = plt.figure(1,figsize=(10, 7))\nplots = []\nfor i in range(3):\n    for j in range(1,4):\n        ind = i*3+j\n        plt.subplot(3,3,ind)\n        plt.axis('off')\n        plt.tight_layout()\n        plots.append(plt.imshow(np.zeros([28,28]),cmap='plasma',clim=(-1, 1)))\n        plt.title(ind-1)\n\ndef refresh_plots(list_of_plots, data):\n    for ind, plot in enumerate(plots):\n        plot.set_data(normalize(data[:,ind]).reshape(28,28))\n    fig.canvas.draw()","execution_count":174,"outputs":[]},{"metadata":{"_cell_guid":"2baa785d-3d2c-4567-9e32-8cefed2cc9d0","_uuid":"51a09eac71524b9c82f840a256f2e66a3f4910fa","colab":{"autoexec":{"startup":false,"wait_interval":0},"base_uri":"https://localhost:8080/","height":1693,"output_extras":[{"item_id":1}]},"colab_type":"code","executionInfo":{"elapsed":560,"status":"error","timestamp":1520193056340,"user":{"displayName":"Alexandre Gomes","photoUrl":"//lh4.googleusercontent.com/-0SJODzHVIXQ/AAAAAAAAAAI/AAAAAAAAUeU/Cs1pVksWH3I/s50-c-k-no/photo.jpg","userId":"108678232453929155174"},"user_tz":0},"id":"GPjo9H-7fEZY","outputId":"e862c660-82d5-4284-edfa-04c8508d41d9","scrolled":false,"trusted":true},"cell_type":"code","source":"N=1\nfor i in range(10):\n    loss.append(sess.run(loss_func, feed_dict={x: test_images,y_: test_labels}))\n    sess.run(train_step, feed_dict={x: train_images, y_: train_labels})\n    if not i%N: #refresh weights plot every N steps\n        refresh_plots(plots, normalize(sess.run(W)))\n        pause(1)\nplt.figure(3)\nloss_function_plt = plt.plot(loss)","execution_count":178,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"097d6ac014566ef0d7abb66a6474b2e5c9093069"},"cell_type":"code","source":"#W.initializer.run()","execution_count":null,"outputs":[]}],"metadata":{"colab":{"default_view":{},"name":"MNIST with Deep NN tutorial.ipynb","provenance":[],"version":"0.3.2","views":{}},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}