{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Pre-processing**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Processing to split train.csv to training and validation datasets; and to create npz files for processing \ntrain_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')\n\ntrain_inputs_all = train_data.iloc[:,1:]\ntrain_targets_all = train_data.iloc[:,0:1] \ntrain_targets_all = pd.get_dummies(train_targets_all.label)\n\nsample_count = train_data.iloc[:,1:].shape[0]\n\ntrain_sample_count = int(0.9*sample_count)\nvalidation_sample_count = sample_count - train_sample_count\n\ntrain_inputs = train_inputs_all.iloc[:train_sample_count]\ntrain_targets = train_targets_all.iloc[:train_sample_count]\n\nvalidation_inputs = train_inputs_all.iloc[train_sample_count:]\nvalidation_targets = train_targets_all.iloc[train_sample_count:]\n\n\n# Any results you write to the current directory are saved as output.","execution_count":178,"outputs":[{"output_type":"stream","text":"['train.csv', 'sample_submission.csv', 'test.csv']\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"**Model**"},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"input_size = 784\noutput_size = 10\n# Use same hidden layer size for all hidden layers\nhidden_layer_size = 1000\n\n# Reset any variables left in memory from previous runs.\ntf.reset_default_graph()\n\n# Declare placeholders where the data will be fed into.\ninputs = tf.placeholder(tf.float32, [None, input_size])\ntargets = tf.placeholder(tf.float32, [None, output_size])\n\n# Weights and biases for the first linear combination between the inputs and the first hidden layer.\n# Use get_variable in order to make use of the default TensorFlow initializer which is Xavier.\nweights_1 = tf.get_variable(\"weights_1\", [input_size, hidden_layer_size])\nbiases_1 = tf.get_variable(\"biases_1\", [hidden_layer_size])\n\n# Operation between the inputs and the first hidden layer.\n# ReLu is the first activation function. \noutputs_1 = tf.nn.relu(tf.matmul(inputs, weights_1) + biases_1)\n\n# Weights and biases for the second linear combination.\n# This is between the first and second hidden layers.\nweights_2 = tf.get_variable(\"weights_2\", [hidden_layer_size, hidden_layer_size])\nbiases_2 = tf.get_variable(\"biases_2\", [hidden_layer_size])\n\n# Operation between the first and the second hidden layers using sigmoid\noutputs_2 = tf.nn.sigmoid(tf.matmul(outputs_1, weights_2) + biases_2)\n\n# Weights and biases for the third linear combination.\n# This is between second and third hidden layers.\nweights_3 = tf.get_variable(\"weights_3\", [hidden_layer_size, hidden_layer_size])\nbiases_3 = tf.get_variable(\"biases_3\", [hidden_layer_size])\n\n# Operation between the second and third hidden layers\noutputs_3 = tf.nn.sigmoid(tf.matmul(outputs_2, weights_3) + biases_3)\n\n# Weights and biases for the fourth linear combination.\n# This is between the third and fourth hidden layers.\nweights_4 = tf.get_variable(\"weights_4\", [hidden_layer_size, hidden_layer_size])\nbiases_4 = tf.get_variable(\"biases_4\", [hidden_layer_size])\n\n# Operation between the third and fourth hidden layers.\noutputs_4 = tf.nn.sigmoid(tf.matmul(outputs_3, weights_4) + biases_4)\n\n# Weights and biases for the fifth linear combination.\n# This is between the fourth and fifth hidden layers.\nweights_5 = tf.get_variable(\"weights_5\", [hidden_layer_size, hidden_layer_size])\nbiases_5 = tf.get_variable(\"biases_5\", [hidden_layer_size])\n\n# Operation between the fourth and fifth hidden layers.\noutputs_5 = tf.nn.sigmoid(tf.matmul(outputs_4, weights_5) + biases_5)\n\n# Weights and biases for the final linear combination.\n# That's between the fifth hidden layer and the output layer.\nweights_6 = tf.get_variable(\"weights_6\", [hidden_layer_size, output_size])\nbiases_6 = tf.get_variable(\"biases_6\", [output_size])\n\n# Operation between the fifth hidden layer and the final output.\n# Haven't used an activation function because it will be included directly in \n# the loss function. This works for softmax and sigmoid with cross entropy.\noutputs = tf.matmul(outputs_5, weights_6) + biases_6\n\n# Calculate the loss function for every output/target pair.\n# The function used is the same as applying softmax to the last layer and then calculating cross entropy\n# Logits here means: unscaled probabilities (so, the outputs, before they are scaled by the softmax)\n# Naturally, the labels are the targets.\nloss = tf.nn.softmax_cross_entropy_with_logits(logits=outputs, labels=targets)\n\n# Get the average loss\nmean_loss = tf.reduce_mean(loss)\n\n# Define the optimization step. Using adaptive optimizers such as Adam in TensorFlow\noptimize = tf.train.AdamOptimizer(learning_rate=0.001).minimize(mean_loss)\n\n# Get a 0 or 1 for every input in the batch indicating whether it output the correct answer out of the 10.\nout_equals_target = tf.equal(tf.argmax(outputs, 1), tf.argmax(targets, 1))\n\n# Get the average accuracy of the outputs.\naccuracy = tf.reduce_mean(tf.cast(out_equals_target, tf.float32))\n\n# Declare the session variable.\nsess = tf.InteractiveSession()\n\n# Initialize the variables. Default initializer is Xavier.\ninitializer = tf.global_variables_initializer()\nsess.run(initializer)\n\n# Batching\nbatch_size = 1000\n\n# Calculate the number of batches per epoch for the training set.\nbatches_number = train_inputs.shape[0] // batch_size\n\n# Basic early stopping. Set a miximum number of epochs.\nmax_epochs = 15\n\n# Keep track of the validation loss of the previous epoch.\n# If the validation loss becomes increasing, we want to trigger early stopping.\n# We initially set it at some arbitrarily high number to make sure we don't trigger it\n# at the first epoch\nprev_validation_loss = 9999999.\n\nimport time\nstart_time = time.time()\n\n# Create a loop for the epochs. Epoch_counter is a variable which automatically starts from 0.\nfor epoch_counter in range(max_epochs):\n    \n    # Keep track of the sum of batch losses in the epoch.\n    curr_epoch_loss = 0.\n    \n    # Iterate over the batches in this epoch.\n    for batch_counter in range(batches_number):\n        \n        # Input batch and target batch are assigned values from the train dataset, given a batch size\n        input_batch = train_inputs.iloc[batch_counter*batch_size:batch_counter*batch_size+batch_size,:]\n        target_batch = train_targets.iloc[batch_counter*batch_size:batch_counter*batch_size+batch_size,:]\n        \n        # Run the optimization step and get the mean loss for this batch.\n        # Feed it with the inputs and the targets we just got from the train dataset\n        _, batch_loss = sess.run([optimize, mean_loss], feed_dict={inputs: input_batch, targets: target_batch})\n        \n        # Increment the sum of batch losses.\n        curr_epoch_loss += batch_loss\n    \n    # So far curr_epoch_loss contained the sum of all batches inside the epoch\n    # We want to find the average batch losses over the whole epoch\n    # The average batch loss is a good proxy for the current epoch loss\n    curr_epoch_loss /= batches_number\n    \n    # At the end of each epoch, get the validation loss and accuracy\n    # Get the input batch and the target batch from the validation dataset\n    input_batch = validation_inputs.iloc[:,:]\n    target_batch = validation_targets.iloc[:,:]\n    \n    \n    # Run without the optimization step (simply forward propagate)\n    validation_loss, validation_accuracy = sess.run([mean_loss, accuracy], \n        feed_dict={inputs: input_batch, targets: target_batch})\n    \n    # Print statistics for the current epoch\n    # Epoch counter + 1, because epoch_counter automatically starts from 0, instead of 1\n    # We format the losses with 3 digits after the dot\n    # We format the accuracy in percentages for easier interpretation\n    print('Epoch '+str(epoch_counter+1)+\n          '. Mean loss: '+'{0:.3f}'.format(curr_epoch_loss)+\n          '. Validation loss: '+'{0:.3f}'.format(validation_loss)+\n          '. Validation accuracy: '+'{0:.2f}'.format(validation_accuracy * 100.)+'%')\n    \n    # Trigger early stopping if validation loss begins increasing.\n    if validation_loss > prev_validation_loss:\n        break\n        \n    # Store this epoch's validation loss to be used as previous validation loss in the next iteration.\n    prev_validation_loss = validation_loss\n\n# Not essential, but it is nice to know when the algorithm stopped working in the output section, rather than check the kernel\nprint('End of training.')\n\n#Add the time it took the algorithm to train\nprint(\"Training time: %s seconds\" % (time.time() - start_time))","execution_count":179,"outputs":[{"output_type":"stream","text":"/opt/conda/lib/python3.6/site-packages/tensorflow/python/client/session.py:1702: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n  warnings.warn('An interactive session is already active. This can '\n","name":"stderr"},{"output_type":"stream","text":"Epoch 1. Mean loss: 1.608. Validation loss: 0.485. Validation accuracy: 87.98%\nEpoch 2. Mean loss: 0.280. Validation loss: 0.207. Validation accuracy: 94.31%\nEpoch 3. Mean loss: 0.150. Validation loss: 0.165. Validation accuracy: 95.50%\nEpoch 4. Mean loss: 0.108. Validation loss: 0.140. Validation accuracy: 96.14%\nEpoch 5. Mean loss: 0.088. Validation loss: 0.134. Validation accuracy: 96.62%\nEpoch 6. Mean loss: 0.068. Validation loss: 0.119. Validation accuracy: 96.74%\nEpoch 7. Mean loss: 0.059. Validation loss: 0.161. Validation accuracy: 95.81%\nEnd of training.\nTraining time: 3.1716034412384033 seconds\n","name":"stdout"}]},{"metadata":{},"cell_type":"markdown","source":"**Model Testing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nout = sess.run([outputs], feed_dict={inputs: test_data})\nout_df = np.row_stack(out)\n\nargmax_outputs = np.argmax(out_df,1)\nargmax_outputs = np.array([argmax_outputs]).T\n#print(argmax_outputs)\n\nimage_id = np.arange(1, len(argmax_outputs) + 1, 1)\nimage_id = image_id.reshape(len(argmax_outputs), 1)\n\nnp.savetxt('submission.csv', np.c_[image_id, argmax_outputs], delimiter=',', header='ImageId,Label', comments='', fmt='%d')\n","execution_count":180,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}