{"cells":[{"metadata":{"_uuid":"54c10d67a435744dea18a908e0e7b0f50118c017"},"cell_type":"markdown","source":"I have created this kernel  as a reference for myself and other beginners to perform image recognition using Convolutional Neural Network using Tensorflow and Python. It contains step by step breakdown of the whole CNN classification from start to finish. The code is based on and referencing the following:\n\nhttps://www.tensorflow.org/tutorials/estimators/cnn\n\nhttps://www.kaggle.com/kakauandme/tensorflow-deep-nn\n\nhttps://www.kaggle.com/flaport/tensorflow-cnn-lb-0-98929\n\nhttps://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/convolutional_network.py\n\nhttps://www.kaggle.com/scolianni/tensorflow-convolutional-neural-network-for-mnist\n\nIf you find this helpful, take a look at these sources for more information."},{"metadata":{"_uuid":"7a9279939f40f8b5d400120e8c641aa91f56a5e4"},"cell_type":"markdown","source":"## 1. Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#Allows using operating system dependent functionality\nimport os\n#Provides support for multidimensional arrays and high-level mathematical functions\nimport numpy as np\n #Provides support for data manipulation and analysis\nimport pandas as pd\n#Provides MATLAB-like plotting framework (based on numpy)\nimport matplotlib.pyplot as plt\n#Provides data visualisation interface for drawing and graphing (based on matplotlib)\nimport seaborn as sns\n#Provides a framework for fast numerical computing, machine learning and neural networks\nimport tensorflow as tf\n#Provides a random permutation cross-validator. Yields indices to split data into training and test sets\nfrom sklearn.model_selection import ShuffleSplit\n#Scales/standartizes features by removing the mean and scaling to unit variance\nfrom sklearn.preprocessing import StandardScaler\n#Encodes labels with value between 0 and n_classes-1\nfrom sklearn.preprocessing import LabelEncoder\n#Encodes categorical integer features using a one-hot aka one-of-K scheme\nfrom sklearn.preprocessing import OneHotEncoder","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Chekcing input directory\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"182c9edb02cf579a36c921428ec3ac47399a713b"},"cell_type":"markdown","source":"## 2. Settings Setup"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"69199869282daf1a9b63e584bbfd7bd5f8030eec"},"cell_type":"code","source":"# Learning rate\nLEARNING_RATE = 0.001\n# Number of training iterations to run\nSTEPS = 25000 \n#Validation data size\nVALIDATION_SIZE = 1000 \n#Types of labels for digits 1 through 10\nLABELS = 10\n#Width/height of the image\nDIMENSION = 28\n#Number of colors in the image (greyscale)\nCOLOUR_CHANNELS = 1\n#Batch size\nBATCH_SIZE = 100\n#Convolutional Kernel size\nCONVOLUTION_PATCH = 5\n#Number of Convolutional Kernels\nDEPTH = 32\n#Number of hidden neurons in the fully connected layer\nHIDDEN = 1024","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f3aee07eaee2281120540e861862a4bd7cedee9"},"cell_type":"markdown","source":"## 3. Data Import and Preprocessing"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"189d8bbde1d28f94679ad87b41e3cddeaa0d9cc9"},"cell_type":"code","source":"#Read cvs files into pandas dataframes\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68234bd537aae3181300b12f78f909607fc36225"},"cell_type":"code","source":"#Look at top 5 rows of train dataframe\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5953d82d0b8a7120540b9bd561ed61f907dff39c","collapsed":true},"cell_type":"code","source":"#Remove the labels as a numpy array from the dataframe\nlabels = np.array(train.pop('label'))\n#Encode labels\nlabels = LabelEncoder().fit_transform(labels)[:, None]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a694d6123aacca6d1ac1c97d18c2805a433f56d"},"cell_type":"code","source":"#Look at encoded labels\nprint(labels.shape)\nprint('\\n')\nprint(labels[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4bb0a81775cd77542c960639f6e6ab8a433f0793"},"cell_type":"code","source":"#Apply OneHot encoding\nlabels = OneHotEncoder().fit_transform(labels).todense()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39262a8fd161eed267ba127d61aca183e303b41e"},"cell_type":"code","source":"#Look at one hot encoded labels\nprint(labels.shape)\nprint('\\n')\nprint(labels[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"67754995aef5882619b9c9d1d4e42480a637a60f"},"cell_type":"code","source":"#Convert the dataframe to a numpy array\ntrain = StandardScaler().fit_transform(np.float32(train.values)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67c8c30c7f9eaf22cc5bd67ef4675642eb0c99c0"},"cell_type":"code","source":"#Look at scaled train dataframe\nprint(train.shape)\nprint('\\n')\nprint(train[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"10433c1044622c104fb780c58372b348bbee5632"},"cell_type":"code","source":"#Reshape the data into 42000 28x28 arrays (Number of images, height, width, colour channels)\ntrain = train.reshape(-1, DIMENSION, DIMENSION, COLOUR_CHANNELS) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"040da2cc1a86d290088dc45ccd629e1cb7ff222f"},"cell_type":"code","source":"#Look at reshaped train dataframe\nprint(train.shape)\nprint('\\n')\nprint(train[:1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46b364240465dce12393fd077ed03c2fc2d4a262","collapsed":true},"cell_type":"code","source":"#Split data into training and validation sets\nts_data, valids_data = train[:-VALIDATION_SIZE], train[-VALIDATION_SIZE:]\nts_labels, valids_labels = labels[:-VALIDATION_SIZE], labels[-VALIDATION_SIZE:]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8427681ad76103cc730f651f17168074e7171fb7"},"cell_type":"markdown","source":"## 4. Build the Model"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"70962554c7427b211a63f26ff068737426316a47"},"cell_type":"code","source":"#Initialize the input data with placeholders.\ntf_data = tf.placeholder(tf.float32, shape=(None, DIMENSION, DIMENSION, COLOUR_CHANNELS))\ntf_labels = tf.placeholder(tf.float32, shape=(None, LABELS))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"86e035baf3360eda196b3bea2812ccf798cfb2e1"},"cell_type":"code","source":"#Initialise the biases\n#Convolutional layer 1 biases with depth of DEPTH\nb1 = tf.Variable(tf.zeros([DEPTH]))\n#Convolutional layer 2 biases (twice the depth of the first convolutional layer)\nb2 = tf.Variable(tf.constant(1.0, shape=[2 * DEPTH]))\n#Hidden/Dense layer biases with HIDDEN of hidden nodes\nb3 = tf.Variable(tf.constant(1.0, shape=[HIDDEN]))\n#Output layer biases\nb4 = tf.Variable(tf.constant(1.0, shape=[LABELS]))\n\n#Initialise the weights with patch size of CONVOLUTION_PATCH\n#Convolutional layer 1 weights\nw1 = tf.Variable(tf.truncated_normal([CONVOLUTION_PATCH, CONVOLUTION_PATCH, COLOUR_CHANNELS, DEPTH], stddev=0.1))\n#Convolutional layer 2 weights (twice the depth of the first convolutional layer)\nw2 = tf.Variable(tf.truncated_normal([CONVOLUTION_PATCH, CONVOLUTION_PATCH, DEPTH, 2 * DEPTH], stddev=0.1))\n#Hidden/Dense layer weights\nw3 = tf.Variable(tf.truncated_normal([DIMENSION // 4 * DIMENSION // 4 * 2 * DEPTH, HIDDEN], stddev=0.1))\n#Output layer biaweightsses\nw4 = tf.Variable(tf.truncated_normal([HIDDEN, LABELS], stddev=0.1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4296bb4b787625a57ba1ab11421d37643d47f664"},"cell_type":"code","source":"#Assemble the layers\ndef logits(input):\n    #Convolutional layer 1\n    mnist_classifier = tf.nn.conv2d(input, w1, [1, 1, 1, 1], padding='SAME')\n    mnist_classifier = tf.nn.max_pool(mnist_classifier, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n    mnist_classifier = tf.nn.relu(mnist_classifier + b1)\n    #Convolutional layer 2\n    mnist_classifier = tf.nn.conv2d(mnist_classifier, w2, [1, 1, 1, 1], padding='SAME')\n    mnist_classifier = tf.nn.max_pool(mnist_classifier, [1, 2, 2, 1], [1, 2, 2, 1], padding='SAME')\n    mnist_classifier = tf.nn.relu(mnist_classifier + b2)\n    #Hidden layer\n    mnist_classifier = tf.reshape(mnist_classifier, (-1, DIMENSION // 4 * DIMENSION // 4 * 2 * DEPTH))\n    mnist_classifier = tf.nn.relu(tf.matmul(mnist_classifier, w3) + b3)\n    return tf.matmul(mnist_classifier, w4) + b4\n\n#Prediction:\ntf_pred = tf.nn.softmax(logits(tf_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a2276e2872f30657af129652da89d56cac60275"},"cell_type":"code","source":"#Get the loss by using Categorical Cross Entropy Loss function for training the model.\ntf_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits(tf_data), labels=tf_labels))\n#Get accuracy \ntf_acc = 100 * tf.reduce_mean(tf.to_float(tf.equal(tf.argmax(tf_pred, 1), tf.argmax(tf_labels, 1))))\n\n#Set up one of the following (or other) optimisers\n\n#tf_opt = tf.train.RMSPropOptimizer(LEARNING_RATE)\ntf_opt = tf.train.AdamOptimizer(LEARNING_RATE)\n\ngrads_and_vars = tf_opt.compute_gradients(tf_loss)\ntf_step = tf_opt.minimize(tf_loss)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"81faafed1f4569b4e5d2f9b32a3e980fcd86f965"},"cell_type":"markdown","source":"## 5. Train the Model"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"5349713210f75df15d6228e19e9234ef9b387cb5"},"cell_type":"code","source":"#Open tensorflow session\ninit = tf.global_variables_initializer()\nsession = tf.Session()\nsession.run(init)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"69322e39325586da8522f3d2791e16560d9527a7"},"cell_type":"code","source":"#Initialise error log\nerror_log = [(0, 0, 10)] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07cd465e2d3ebea52d9ad14dd78a0d4e4efeb17e","scrolled":true},"cell_type":"code","source":"#Run the session\nss = ShuffleSplit(n_splits=STEPS, train_size=BATCH_SIZE)\n#Get the number of splitting iterations in the cross-validator\nss.get_n_splits(ts_data, ts_labels)\n#Go through the data for STEPS of steps\nfor step, (id, _) in enumerate(ss.split(ts_data,ts_labels), start=1):\n    #Get single image and its label at id of id and set it to fd\n    fd = {tf_data:ts_data[id], tf_labels:ts_labels[id]}\n    #Run the session for single image with the chosen optimiser\n    session.run(tf_step, feed_dict=fd)\n    #Every 500 steps do this\n    if step%1000 == 0:\n        #Get validation images and labels and assign them to fd\n        fd = {tf_data:valids_data, tf_labels:valids_labels}\n        #Get model loss and accuracy from the validation dataset\n        validation_loss, validation_accuracy = session.run([tf_loss, tf_acc], feed_dict=fd)\n        #Save model loss and accuracy at each 500 step to history\n        error_log.append((step, validation_loss, validation_accuracy))\n        #Print step and accuracy\n        print('Step %i \\t Valid. Acc. = %f'%(step, validation_accuracy), end='\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c533cdc72316ca2101e64e9fd65a18f5b927f16f"},"cell_type":"code","source":"#Unzip the history list into 3 tuples\nsteps, loss, accuracy = zip(*error_log)\n#Create a figure and set plot size\nfig = plt.figure(figsize=(16,6))\n#Add first subplot\nsub1 = fig.add_subplot(221)\n#Plot the data on first subplot\nsub1.plot(steps,loss, 'o-')\n#Set title for first subplot\nsub1.set_title('Validation Loss')\n#Set x label for first subplot\nsub1.set_xlabel('Steps')\n#Set y label for first subplot\nsub1.set_ylabel('Log(Loss)')\n#Add second subplot\nsub2 = fig.add_subplot(222)\n#Plot the data on second subplot\nsub2.plot(steps, accuracy, '.-')\n#Set x label for second subplot\nsub2.set_xlabel('Steps')\n#Set y label for second subplot\nsub2.set_ylabel('Accuracy')\n#Set title for second subplot\nsub2.set_title('Accuracy')\n#Make sure plots don't overlap\nplt.tight_layout()\n#Show plots\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7545de9cff0e72bd967177a0b2b398cc3d92514a"},"cell_type":"markdown","source":"## 6. Classify Test Data"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0cad6ead23c0f9e062d410409a563ccef8ffa0b6"},"cell_type":"code","source":"# Convert the test dataframe to a numpy array\ntest_data = StandardScaler().fit_transform(np.float32(test.values)) \n#Reshape test_data into 42000 28x28 matricies\ntest_data = test_data.reshape(-1, DIMENSION, DIMENSION, COLOUR_CHANNELS) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb92725a925c4db3534318a2ea236bc0cbddec84","collapsed":true},"cell_type":"code","source":"#Make a prediction about the test labels\ntest_pred = session.run(tf_pred, feed_dict={tf_data:test_data})\ntest_labels = np.argmax(test_pred, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6dc88df4c48339e3ae44461451cb68575279acd"},"cell_type":"code","source":"#Show the structure of the predictions\nprint(test_pred.shape)\nprint(test_labels.shape)\nprint(test_pred[:2])\nprint(test_labels[:2])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"186e16599e29887adf6cb2f17ef0c5fda0513f43"},"cell_type":"markdown","source":"## 7. Check and Save the Predictions"},{"metadata":{"trusted":true,"_uuid":"6940fe17383952922fa9235f6c1b983e5bf68597"},"cell_type":"code","source":"#Plotting an example\nimage = 34\nplt.axis('off')\nplt.imshow(test_data[image,:,:,0])\nplt.show()\nprint(\"Prediction: %i\"%test_labels[image])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ba42ead5d8db5947eab35667d4a44d4f7458a39"},"cell_type":"code","source":"#Save predictions to a dataframe\npredictions = pd.DataFrame(data={'ImageId':(np.arange(test_labels.shape[0])+1), 'Label':test_labels})\n#Write the datafram to a csv file\npredictions.to_csv('predictions2.csv', index=False)\n#Show sample of the submission dataframe\npredictions.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d43c04c6de4cd98cdd8fbd1377e0bf54f2c54357"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}