{"cells":[{"metadata":{"trusted":true,"_uuid":"755b7312768314cbf9cadf72689c26d8825e8aad"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport math\nfrom tensorflow.python.framework import ops\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":1,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"5969db94537c873fe464b3e66f05b08f23a6803b"},"cell_type":"code","source":"# load the data and assign to X and y\ntrain = pd.read_csv('../input/train.csv')\nX_test = pd.read_csv('../input/test.csv')\n\nX_train = train.drop(labels = [\"label\"],axis = 1) \ny_train = train[\"label\"]\n\n# Now normalize the test and train sets\nX_train = X_train / 255.0\nX_test = X_test / 255.0\n\n# Reshape image in 3 dimensions (height = 28px, width = 28px)\nX_train = X_train.values.reshape(-1,28,28,1)\nX_test = X_test.values.reshape(-1,28,28,1)","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc4208930cfcf0aa42b5fbc705593b84c02ac833"},"cell_type":"code","source":"# What are the sizes of the train and test sets?\nprint('train size: ' + str(len(X_train)) + ', test size: ' + str(len(X_test)))","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"994d601f307f342e1e9069ee20b2d0ba25016b72"},"cell_type":"code","source":"# So we have 42,000 train images and 28,000 test images\n# let's go ahead and print one\nprint(X_train[0].shape)","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83943b2080ddf4ac740505b2f4591ba3deb4eb6a"},"cell_type":"code","source":"# Let's now reshape it into a 28 x 28 matrix\nimage = X_train[9]\nprint(image.shape)\n\n# Let's plot it now\nplt.imshow(image.reshape(28,28))","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7e7ce3385c0ebd44b4d1ac18f1446dbf41a04be"},"cell_type":"code","source":"# Now let's look at the labels\nprint(y_train.values.shape)\ny_train.values.reshape(-1, 1)","execution_count":6,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"2c3b9ff9a6a85eabb36825599fc217c059e9e84d"},"cell_type":"code","source":"# So the labels are NOT one hot encoded. We need to do that\nfrom sklearn.preprocessing import OneHotEncoder\nenc = OneHotEncoder(sparse=False)\ny_train_ohe = enc.fit_transform(y_train.values.reshape(-1, 1))","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ddb5951e4ee961f595e496262833dc6584241d3c"},"cell_type":"code","source":"y_train_ohe.shape","execution_count":8,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"d1ca197f81aaac6ff8fc7d453b3503aebb4cf210"},"cell_type":"code","source":"# Let's first reset our graph, so our neural network components are all declared within the same graph\nops.reset_default_graph()","execution_count":9,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"291e3420b3405ed844e11bf600274309e5388f54"},"cell_type":"code","source":"# The input, X, is 28 x 28 grayscale (1 color channel) images, and\n# I'll break them up into mini-batches of an unknown size at this\n# point -- thus the 'None' as the 1st dimension\nX = tf.placeholder(tf.float32, [None, 28, 28, 1])\n# y_hat, estimates output by the softmax layer\nY_ = tf.placeholder(tf.float32, [None, 10])\n# we'll do a variable learning rate\nlr = tf.placeholder(tf.float32)","execution_count":10,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"988b47083153632e220613906abd7c035d023eb6"},"cell_type":"code","source":"# The number of channels (aka. kernels or filters) that will be\n# applied in each convolutional layer\nc1 = 6 # first convolutional layer\nc2 = 12 # second convolutional layer\nc3 = 24 # third convolutional layer\n\n# the size of each filter (5 by 5)\nfilter_size = 5\n\n# then we'll have one fully connected layer with 200 nodes\nf1 = 200","execution_count":11,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"8a662125629ad3f6ed735aed8e1a31d7554be234"},"cell_type":"code","source":"# Time to setup the weights and biases associated with each layer.\n# Note how much smaller the number of parameters is as compared with\n# a fully connected NN with similar number of layers.\n\n# We'll set W1 with positive values from a Gaussian distribution\n# having standard deviation of 0.1\n\n# TODO fully understand this matrix math here\nW1 = tf.Variable(tf.truncated_normal([filter_size, filter_size, 1, c1], stddev=0.1))\nB1 = tf.Variable(tf.ones([c1])/10)\nW2 = tf.Variable(tf.truncated_normal([filter_size, filter_size, c1, c2], stddev=0.1))\nB2 = tf.Variable(tf.ones([c2])/10)\nW3 = tf.Variable(tf.truncated_normal([filter_size, filter_size, c2, c3], stddev=0.1))\nB3 = tf.Variable(tf.ones([c3])/10)\n\n# Now the weights and biases for the fully connected layer. It's\n# 7 * 7 * 24 by 200 because the activation matrix after the final\n# convolutional layer is 7 x 7 x 24. So all 200 nodes receive 1,176\n# inputs.\nW4 = tf.Variable(tf.truncated_normal([7 * 7 * c3, f1], stddev=0.1))\nB4 = tf.Variable(tf.ones([f1])/10)\n\n# Now for the softmax (output) layer\nW5 = tf.Variable(tf.truncated_normal([f1, 10], stddev=0.1))\nB5 = tf.Variable(tf.ones([10])/10)","execution_count":12,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"2aab7af962b2b225823d40b053d72c2fecc9e0a0"},"cell_type":"code","source":"# Time to build the model! We will perform same padding for each\n# convolution, reducing dimensions instead by specifying a stride\n# of two for Conv layers 2 and 3\nstride = 1  # output is 28x28\nY1 = tf.nn.relu(tf.nn.conv2d(X, W1, strides=[1, stride, stride, 1], padding='SAME') + B1)\nstride = 2  # output is 14x14\nY2 = tf.nn.relu(tf.nn.conv2d(Y1, W2, strides=[1, stride, stride, 1], padding='SAME') + B2)\nstride = 2  # output is 7x7\nY3 = tf.nn.relu(tf.nn.conv2d(Y2, W3, strides=[1, stride, stride, 1], padding='SAME') + B3)\n\n# Flatten the output of the third convolutional layer to go into\n# the fully connected layer\nYY = tf.reshape(Y3, shape=[-1, 7 * 7 * c3])\n\n# Activation for fully connected layer\nY4 = tf.nn.relu(tf.matmul(YY, W4) + B4)\n\n# softmax layer\nYlogits = tf.matmul(Y4, W5) + B5\nY = tf.nn.softmax(Ylogits)","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0186317be8a3a2221deff4a2118c666cc674bc8"},"cell_type":"code","source":"# cross-entropy loss function (= -sum(Y_i * log(Yi)) ), normalised for batches of 100  images\n# TensorFlow provides the softmax_cross_entropy_with_logits function to avoid numerical stability\n# problems with log(0) which is NaN\ncost = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)\ncost = tf.reduce_mean(cost)\n\n# accuracy of the trained model, between 0 (worst) and 1 (best)\ncorrect_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n\n# prediction of the trained model, for classifying the test set\nprediction = tf.argmax(Y, 1)\n\n# training step, the learning rate is a placeholder\ntrain_step = tf.train.AdamOptimizer(lr).minimize(cost)","execution_count":14,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"d0f590160f2a076419ed3a19a1769a83393fa348"},"cell_type":"code","source":"# now let's initialize and run the session!\ninit = tf.global_variables_initializer()\n\n# TODO here I am!\nimport time\nfrom sklearn.model_selection import train_test_split","execution_count":16,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"c96c6303b9d22f53caeb936868cb0bbb947f3b6a"},"cell_type":"code","source":"def train(dev_split=0.2, batch_size=500, learning_rate=0.003, \n          epochs=5, predict=True):\n    \n    training_start = time.time()\n    \n    with tf.Session() as sess:\n        \n        # Initialize the variables\n        sess.run(init)\n        \n        # number of batches in an epoch\n        batches = int((len(X_train) * (1-dev_split)) // batch_size)\n        print(\"Training\", epochs, \"epochs with\", batches, \"batches per epoch.\")\n        \n        for epoch in range(epochs):\n            \n            epoch_start = time.time()\n            epoch_cost = 0\n            \n            # split the data into a train and dev set for each epoch\n            x_train_, x_dev, y_train_, y_dev = train_test_split(X_train, y_train_ohe, test_size=dev_split)\n            \n            # run this training data through the neural network\n            for batch in range(batches):\n                \n                # Calculate the start and end indices for the next batch\n                begin = (batch * batch_size)\n                end   = (batch * batch_size) + batch_size\n\n                # Get the next sequential batch from the training data\n                batch_xs, batch_ys = x_train_[begin:end], y_train_[begin:end]\n                \n                # Feed this batch through the neural network.\n                _, batch_cost = sess.run([train_step, cost], feed_dict={X: batch_xs, Y_: batch_ys, lr: learning_rate})\n                # print(batch_cost)\n                epoch_cost += batch_cost\n            \n            # calculate accuracy and print results of epoch\n            train_accuracy, train_cost = sess.run([accuracy, cost], \n                                                  {X: x_train_, Y_: y_train_})\n            dev_accuracy, dev_cost = sess.run([accuracy, cost], \n                                                {X: x_dev, Y_: y_dev})\n            epoch_time = round(time.time() - epoch_start, 1)\n            print(\"Epoch\", epoch+1, \"-\", epoch_time, \"s - train/dev cost:\", \n                  round(train_cost, 4), \"|\", round(dev_cost, 4), \n                  \"- train/dev accuracy:\", round(train_accuracy, 4), \n                  \"|\", round(dev_accuracy, 4))\n        \n        # Generate the predictions on the test set\n        if predict:\n            predictions = prediction.eval({X: X_test})\n        \n    print(\"Total training time:\", round(time.time() - training_start, 1), \"s\")\n    \n    # Return the predictions\n    if predict:\n        return predictions\n        ","execution_count":22,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5415157eaaf15a30e8becd1b89c3af0a9b306a8b"},"cell_type":"code","source":"predictions = train()","execution_count":18,"outputs":[]},{"metadata":{"_uuid":"70b7ea2d3d1b511518ecfce0358fdc06768a4240"},"cell_type":"markdown","source":"Pretty good accuracy! Now let's submit our predictions.\n\n### Submitting predictions\n\nThe predictions generated by the model is an array of length 28,000:"},{"metadata":{"trusted":true,"_uuid":"f46181b95ddcee67de6cc442927fb215c01b2275"},"cell_type":"code","source":"predictions.shape","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"5e50ea7f77486a53efbbeb6fc98ad894bf036b4a"},"cell_type":"markdown","source":"Per the competition instructions, we need to submit a .csv file in the format of:\n\n```\nImageId,Label\n1,0\n2,0\n3,0\n...\n```\n\nTo help with this I'll import pandas, create a dataframe with these column headers, and save it to .csv."},{"metadata":{"trusted":true,"_uuid":"7b4fcb47f23ec2dcfee525c7e5d83156fee1e2d2"},"cell_type":"code","source":"import pandas as pd\n\ndf_predictions = pd.DataFrame({'ImageId' : np.arange(len(predictions))+1,\n                               'Label' : predictions})\ndf_predictions.head(10)","execution_count":21,"outputs":[]},{"metadata":{"collapsed":true,"trusted":true,"_uuid":"663e1c7cb224f99a009997a5fa873c77646aab1c"},"cell_type":"code","source":"df_predictions.to_csv('tf_cnn_v1_submission.csv', index=False)","execution_count":36,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}