{"nbformat_minor": 0, "nbformat": 4, "metadata": {"kernelspec": {"name": "python3", "display_name": "Python 3", "language": "python"}, "language_info": {"name": "python", "version": "3.6.1", "codemirror_mode": {"name": "ipython", "version": 3}, "mimetype": "text/x-python", "file_extension": ".py", "nbconvert_exporter": "python", "pygments_lexer": "ipython3"}}, "cells": [{"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "_uuid": "76a90dd72218e9f40094ae5336fbf219739e5f5e", "_cell_guid": "5e965038-03bc-48a8-86bb-733f18b4fc21", "_execution_state": "idle"}, "source": "", "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "6b189b699ac8281d71d960186fa5509e4070dff3", "_cell_guid": "a57859ba-8b07-4e8d-9371-ab44027b2559", "_execution_state": "idle"}, "source": "import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\nimport tensorflow as tf\nimport time\nfrom datetime import timedelta", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "0c03b29d30fee021b1d0c8a39ae3d514a5004dc7", "_cell_guid": "d76dd6c2-70ba-4146-866e-6d935b8c3d62", "_execution_state": "idle"}, "source": "#loading the data sets from the csv files\nprint('--------load train & test file------')\n# read training data from CSV file \ntrain_dataset = pd.read_csv('../input/train.csv')\nprint(train_dataset.shape)\ntrain_dataset = train_dataset.as_matrix()\n\n#test dataset\ntest_dataset = pd.read_csv('../input/test.csv')\nprint(test_dataset.shape)\ntest_dataset = test_dataset.as_matrix()\nprint(\"------finish loading --------------------\")", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "97e94a3fe6b23b3640125d4d3818b34dd2f21397", "_cell_guid": "69bcff74-8245-4d96-9ddc-19a699707695", "_execution_state": "idle"}, "source": "validation_size = 2000\npixel_depth = 255.0  # Number of levels per pixel.\nimage_width = 28\nimage_height = 28", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "dc315a213787874a3fb88100108d4ff5fd938f7a", "_cell_guid": "dc75bb2d-0b43-4d30-ba78-63c0a5804af8", "_execution_state": "idle"}, "source": "#Shuffle the datasets\nfrom sklearn.utils import shuffle\ntrain_dataset = shuffle(train_dataset)\nprint ('finish shuffling the data')", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "e00b9f2cd65f379315cecaf126efc0a4900005d4", "_cell_guid": "f725e18e-2892-4f0c-9e00-499b0d54f5cc", "_execution_state": "idle"}, "source": "dataset_labels = train_dataset[: ,:1]\ndataset_images = train_dataset[: ,1:]\nprint(dataset_labels.shape)\nprint(dataset_images.shape)", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "2dd70b342677d68e8713205b04a492c5418118aa", "_cell_guid": "83e230e9-842c-41ad-a245-8d745fa36ef1", "_execution_state": "idle"}, "source": "from sklearn.preprocessing import OneHotEncoder\n# Fit the OneHotEncoder\nenc = OneHotEncoder().fit(dataset_labels.reshape(-1, 1))\n# Transform the label values to a one-hot-encoding scheme\ndataset_labels = enc.transform(dataset_labels.reshape(-1, 1)).toarray()\n\nprint(\"Training set\", dataset_labels.shape)\nprint ('labels[{0}] => {1}'.format(1,dataset_labels[1]))", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "0da66528fd2dfc367b17dffcd5b20961d0890024", "_cell_guid": "c3aa1ee4-9c46-4352-87d6-6084efe14939", "_execution_state": "idle"}, "source": "del train_dataset", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "ad6858af51a05bc0876e9f9f4b5490355510569f", "_cell_guid": "6f994485-4d16-4610-8954-de5bf8c48aca", "_execution_state": "idle"}, "source": "# split data into dataset_images & validation\nX_val = dataset_images[:validation_size]\ny_val = dataset_labels[:validation_size]\n\nX_train = dataset_images[validation_size:]\ny_train = dataset_labels[validation_size:]", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "db50b4b0bd069062b2a25cf9f56093f1e24eb4d5", "_cell_guid": "0de51957-c44b-4c9f-85da-883b15e76872", "_execution_state": "idle"}, "source": "### Let's apply normalization to our images.\nX_train = (X_train.astype(float) - \n                    pixel_depth / 2) / pixel_depth\n\nX_val = (X_val.astype(float) - \n                    pixel_depth / 2) / pixel_depth\n\nX_test = (test_dataset.astype(float) - \n                    pixel_depth / 2) / pixel_depth", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "1a5b47e1d1b7c0db887577fda35118a225b134a3", "_cell_guid": "9a1b7809-7b47-4dab-9a0e-ca3994ee09bb", "_execution_state": "idle"}, "source": "del dataset_images\ndel dataset_labels\ndel test_dataset", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "6390d818dd710e604f92fbb47bdfb1c00e398cbb", "_cell_guid": "ff633dfd-d96b-4e3b-9d11-6b503b87bd14", "_execution_state": "idle"}, "source": "#reshpe the array \nX_train = X_train.reshape(-1, image_width, image_height, 1)\nX_val =  X_val.reshape(-1, image_width, image_height, 1)\nX_test =  X_test.reshape(-1, image_width, image_height, 1)\n\nprint('Training set', X_train.shape, y_train.shape)\nprint('Validation set', X_val.shape, y_val.shape)\nprint('Test set', X_test.shape)", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "b210d3ffd9a472ac4c58ca278a1709bd7d6779de", "_cell_guid": "c318264c-1788-4ca3-914e-ecedbfe21f23", "_execution_state": "idle"}, "source": "def plot_images(images, nrows, ncols, cls_true, cls_pred=None):\n    \"\"\" Plot nrows * ncols images from images and annotate the images\n    \"\"\"\n    # Initialize the subplotgrid\n    fig, axes = plt.subplots(nrows, ncols)\n    \n    # Randomly select nrows * ncols images\n    rs = np.random.choice(images.shape[0], nrows*ncols)\n    \n    # For every axes object in the grid\n    for i, ax in zip(rs, axes.flat): \n        \n        # Predictions are not passed\n        if cls_pred is None:\n            title = \"T: {0}\".format(np.argmax(cls_true[i]))\n        \n        # When predictions are passed, display labels + predictions\n        else:\n            title = \"T: {0}, P: {1}\".format(np.argmax(cls_true[i]), cls_pred[i])  \n            \n        # Display the image\n        ax.imshow(images[i,:,:,0], cmap='binary')\n        #ax.imshow(images[i].reshape(image_width,image_height), cmap='binary')\n        \n        # Annotate the image\n        ax.set_title(title)\n        \n        # Do not overlay a grid\n        ax.set_xticks([])\n        ax.set_yticks([])\n    fig.show()", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "e9f29a8c16e79ebc4e5d5bd34d7e4da0d6be0c80", "_cell_guid": "73958b44-d162-43b3-baee-a2c5fd4e3424", "_execution_state": "idle"}, "source": "### Plot 2 rows with 9 images each from the training set\nplot_images(X_train, 2, 9, y_train);", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "26b8d92db778b499e881eb9dbb08ae3956cc6c92", "_cell_guid": "a726621b-e806-43f6-8353-908cf93261ef", "_execution_state": "idle"}, "source": "# We know that SVHN images have 32 pixels in each dimension\nimg_size = X_train.shape[1] #28\n# Greyscale images only have 1 color channel\nnum_channels = X_train.shape[-1] # 1 gray\n# Number of classes, one class for each of 10 digits\nnum_classes = y_train.shape[1] # 10 0-9\n\nprint(img_size)\nprint(num_channels)\nprint(num_classes)", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "18e27cfb4b45521ac41a853513b999d16d3114b0", "_cell_guid": "5107d44c-20b3-4c54-ae66-2709edf0c05b", "_execution_state": "idle"}, "source": "def conv_weight_variable(layer_name, shape):\n    \"\"\" Retrieve an existing variable with the given layer name \n    \"\"\"\n    return tf.get_variable(layer_name, shape=shape, initializer=tf.contrib.layers.xavier_initializer_conv2d())\n\ndef fc_weight_variable(layer_name, shape):\n    \"\"\" Retrieve an existing variable with the given layer name\n    \"\"\"\n    return tf.get_variable(layer_name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n\ndef bias_variable(shape):\n    \"\"\" Creates a new bias variable\n    \"\"\"\n    return tf.Variable(tf.constant(0.0, shape=shape))\n\n\ndef conv_layer(input,               # The previous layer\n                layer_name,         # Layer name\n                num_input_channels, # Num. channels in prev. layer\n                filter_size,        # Width and height of each filter\n                num_filters,        # Number of filters\n                pooling=True):      # Use 2x2 max-pooling\n\n    # Shape of the filter-weights for the convolution\n    shape = [filter_size, filter_size, num_input_channels, num_filters]\n\n    # Create new filters with the given shape\n    weights = conv_weight_variable(layer_name, shape=shape)\n    \n    # Create new biases, one for each filter\n    biases = bias_variable(shape=[num_filters])\n\n    # Create the TensorFlow operation for convolution\n    layer = tf.nn.conv2d(input=input,\n                         filter=weights,\n                         strides=[1, 1, 1, 1],\n                         padding='SAME') # with zero padding\n\n    # Add the biases to the results of the convolution\n    layer += biases\n    \n    # Rectified Linear Unit (RELU)\n    layer = tf.nn.relu(layer)\n\n    # Down-sample the image resolution?\n    if pooling:\n        layer = tf.nn.max_pool(value=layer,\n                               ksize=[1, 2, 2, 1],\n                               strides=[1, 2, 2, 1],\n                               padding='SAME')\n\n    # Return the resulting layer and the filter-weights\n    return layer, weights\n\n\ndef flatten_layer(layer):\n    # Get the shape of the input layer.\n    layer_shape = layer.get_shape()\n\n    # The number of features is: img_height * img_width * num_channels\n    num_features = layer_shape[1:4].num_elements()\n    \n    # Reshape the layer to [num_images, num_features].\n    layer_flat = tf.reshape(layer, [-1, num_features])\n\n    # Return the flattened layer and the number of features.\n    return layer_flat, num_features\n\n\n\ndef fc_layer(input,        # The previous layer\n             layer_name,   # The layer name\n             num_inputs,   # Num. inputs from prev. layer\n             num_outputs,  # Num. outputs\n             relu=True):   # Use RELU?\n\n    # Create new weights and biases.\n    weights = fc_weight_variable(layer_name, shape=[num_inputs, num_outputs])\n    biases = bias_variable(shape=[num_outputs])\n\n    # Calculate the layer activation\n    layer = tf.matmul(input, weights) + biases\n\n    # Use ReLU?\n    if relu:\n        layer = tf.nn.relu(layer)\n\n    return layer\n\n# Convolutional Layer 1.\nfilter_size1 = 5          # Convolution filters are 5 x 5 pixels.\nnum_filters1 = 16         # \n\n# Convolutional Layer 2.\nfilter_size2 = 5          # Convolution filters are 5 x 5 pixels.\nnum_filters2 = 32         # \n\n# Fully-connected layer.\nfc_size = 64\n\n\nx = tf.placeholder(tf.float32, shape=(None, img_size, img_size, num_channels), name='x')\ny_true = tf.placeholder(tf.float32, shape=[None, 10], name='y_true')\n\ny_true_cls = tf.argmax(y_true, dimension=1)\nkeep_prob = tf.placeholder(tf.float32)\n\nconv_1, w_c1 = conv_layer(input=x,\n                          layer_name=\"conv_1\",\n                          num_input_channels=num_channels,\n                          filter_size=filter_size1,\n                          num_filters=num_filters1, pooling=True)\n\nconv_1\nconv_2, w_c2 = conv_layer(input=conv_1,\n                          layer_name=\"conv_2\",\n                          num_input_channels=num_filters1,\n                          filter_size=filter_size2,\n                          num_filters=num_filters2,\n                          pooling=True)\n\n# Apply dropout after the pooling operation\ndropout = tf.nn.dropout(conv_2, keep_prob)\n\ndropout\n\n\nlayer_flat, num_features = flatten_layer(dropout)\n\nlayer_flat\n\n\nfc_1 = fc_layer(input=layer_flat,\n                layer_name=\"fc_1\",\n                num_inputs=num_features,\n                num_outputs=fc_size,\n                relu=True)\n\nfc_1\n\nfc_2 = fc_layer(input=fc_1,\n                layer_name=\"fc_2\",\n                num_inputs=fc_size,\n                num_outputs=num_classes,\n                relu=False)\n\nfc_2\n\ny_pred = tf.nn.softmax(fc_2)\n\n# The class-number is the index of the largest element.\ny_pred_cls = tf.argmax(y_pred, dimension=1)\n\n# Calcualte the cross-entropy\ncross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=fc_2, labels=y_true)\n\n# Take the average of the cross-entropy for all the image classifications.\ncost = tf.reduce_mean(cross_entropy)\n\n# Global step is required to compute the decayed learning rate\nglobal_step = tf.Variable(0)\n\n# Apply exponential decay to the learning rate\nlearning_rate = tf.train.exponential_decay(0.05, global_step, 10000, 0.96, staircase=True)\n\n# Construct a new Adam optimizer\noptimizer = tf.train.AdagradOptimizer(learning_rate).minimize(cost, global_step=global_step)\n\n# Predicted class equals the true class of each image?\ncorrect_prediction = tf.equal(y_pred_cls, y_true_cls)\n\n# Cast predictions to float and calculate the mean\naccuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n\nsession = tf.Session()\nsession.run(tf.global_variables_initializer())\n\n# Number of training samples in each iteration \nbatch_size = 64\n\n# Keep probability in dropout layer\ndropout = 0.5\n\ntotal_iterations = 0\n\ndef optimize(num_iterations, display_step):\n    \n    # Ensure we update the global variable rather than a local copy.\n    global total_iterations\n\n    # Start-time used for printing time-usage below.\n    start_time = time.time()\n\n    for step in range(num_iterations):\n\n        offset = (step * batch_size) % (y_train.shape[0] - batch_size)\n        batch_data = X_train[offset:(offset + batch_size), :, :, :]\n        batch_labels = y_train[offset:(offset + batch_size), :]\n        \n        feed_dict_train = {x: batch_data, y_true: batch_labels, keep_prob: dropout}\n\n        # Run the optimizer using this batch of training data.\n        session.run(optimizer, feed_dict=feed_dict_train)\n\n        # Print status every display_step\n        if step % display_step == 0:\n            \n            # Calculate the accuracy on the training-set.\n            batch_acc = session.run(accuracy, feed_dict=feed_dict_train)\n            print(\"Minibatch accuracy at step %d: %.4f\" % (step, batch_acc))\n            \n            # Calculate the accuracy on the validation-set\n            validation_acc = session.run(accuracy, {x: X_val, y_true: y_val, keep_prob: 1.0})\n            print(\"Validation accuracy: %.4f\" % validation_acc)\n\n    # Update the total number of iterations performed.\n    total_iterations += num_iterations\n\n    # Difference between start and end-times.\n    time_diff = time.time() - start_time\n    \n    print(\"Time usage: \" + str(timedelta(seconds=int(round(time_diff)))))\n    \n", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "1688d50c29d9a0c51ca32b44a48d1d9188a72fb3", "_cell_guid": "8b84a21f-1e3b-454e-b5bf-4f25f71dd482", "_execution_state": "idle"}, "source": "# increase num_iterations to get better accuracy = 5000 to 50000 &  display_step=500\noptimize(num_iterations=101, display_step=100)", "cell_type": "code"}, {"execution_count": null, "outputs": [], "metadata": {"collapsed": false, "trusted": false, "_uuid": "d3c86efa6cf9a20e9814f164c99ed75dd2d803b1", "_cell_guid": "4f633215-c168-461d-ab6f-f6c5cb78b38a", "_execution_state": "idle"}, "source": "##predict the test result\n#test_pred = session.run(y_pred_cls, {x: X_test, keep_prob: 1.0})\n#print(test_pred[:10])\n\n## save results\n#np.savetxt('result.csv', np.c_[range(1,len(X_test)+1),test_pred], delimiter=',', header = 'ImageId,Label', comments = '', fmt='%d')\n", "cell_type": "code"}]}