# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn.model_selection import train_test_split
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
#print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
df = pd.read_csv("../input/train.csv")
#train, val = train_test_split(df)
train = df
train_x = train.drop("label", axis=1).as_matrix()
#val_x = val.drop("label", axis=1).as_matrix()

train_x = train_x/255.0
#val_x = val_x/255.0

train_y = train["label"].as_matrix().reshape(-1,1)
#val_y = val["label"].as_matrix().reshape(-1,1)
# binary encode
from sklearn.preprocessing import OneHotEncoder
onehot_encoder = OneHotEncoder(n_values=10, sparse=False)
train_y = onehot_encoder.fit_transform(train_y)
#val_y = onehot_encoder.fit_transform(val_y)

import tensorflow as tf

# Initialise placeholder for training data, 784 elemements to store 28*28 pixel flattened images
X = tf.placeholder(tf.float32, [None, 784])

K = 4
L = 8
M = 12

# Iniitalise variable for storing weights
W1 = tf.Variable(tf.truncated_normal([5, 5, 1, K], stddev=0.1)) #784 x 200
b1 = tf.Variable(tf.ones([K])/10)

W2 = tf.Variable(tf.truncated_normal([5, 5, K, L], stddev=0.1)) # 200 x 100
b2 = tf.Variable(tf.ones([L])/10)

W3 = tf.Variable(tf.truncated_normal([4, 4, L, M], stddev=0.1)) # 100 x 60
b3 = tf.Variable(tf.zeros([M])/10)

N = 200

W4 = tf.Variable(tf.truncated_normal([7*7*M, N], stddev=0.1)) # 60 x 30
b4 = tf.Variable(tf.ones([N])/10)

W5 = tf.Variable(tf.truncated_normal([N, 10], stddev=0.1)) # 30 x 10
b5 = tf.Variable(tf.zeros([10])/10)

init = tf.global_variables_initializer()

X = tf.reshape(X, [-1, 28, 28, 1])
Y1 = tf.nn.relu(tf.nn.conv2d(X, W1, strides=[1,1,1,1], padding='SAME') + b1)
Y2 = tf.nn.relu(tf.nn.conv2d(Y1, W2, strides=[1,2,2,1], padding='SAME') + b2)
Y3 = tf.nn.relu(tf.nn.conv2d(Y2, W3, strides=[1,2,2,1], padding='SAME') + b3)
YY = tf.reshape(Y3, shape=[-1, 7*7*M])

Y4 = tf.nn.sigmoid(tf.matmul(YY, W4) + b4) # using relu for all layers resulted in NaN errors
Y = tf.nn.softmax(tf.matmul(Y4, W5) + b5)

Y_ = tf.placeholder(tf.float32, [None, 10])

# Loss function
cross_entropy = -tf.reduce_sum(Y_ * tf.log(Y))

# % correct answers found in batch
is_correct = tf.equal(tf.argmax(Y,1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))

# Using gradient descent with step size 0.003 with cross entropy loss funciton
optimizer = tf.train.GradientDescentOptimizer(0.003)
train_step = optimizer.minimize(cross_entropy)

# Start TensorFlow session and initalise
sess = tf.Session()
sess.run(init)
listA1 = []
listA2 = []

def next_batch(num, data, labels):

    '''
    Return a total of `num` random samples and labels. 
    '''
    idx = np.arange(0 , len(data))
    np.random.shuffle(idx)
    idx = idx[:num]
    data_shuffle = data[idx]
    labels_shuffle = labels[idx]
   
    return data_shuffle, labels_shuffle
    
for i in range(10000):
    # Get the next set of 100 images to use as training data
    batch_X, batch_Y = next_batch(100, train_x, train_y)
    train_data={X: batch_X.reshape((len(batch_X), 28, 28, 1)), Y_: batch_Y}
    
    # Run training on train data
    sess.run(train_step, feed_dict=train_data)
    
    # # Get accuracy and cross_entropy
    # a1,c = sess.run([accuracy, cross_entropy], feed_dict=train_data)
    # listA1.append(a1)
    # if(i%1000 == 0):
    #     test_data = {X:val_x.reshape((len(val), 28, 28, 1)), Y_:val_y}
    #     a2,c = sess.run([accuracy, cross_entropy], feed_dict=test_data)
    #     listA2.append(a2)
    #     print("i: {:d} a: {:f} c: {:f}".format(i,a2,c))
    # else:
    #     listA2.append(a2)
        
test_x = pd.read_csv("../input/test.csv")
test_x = test_x.as_matrix()/255;
test = {X: test_x.reshape(-1, 28, 28, 1)}

pred = sess.run(tf.argmax(Y, 1), feed_dict=test)
df = pd.DataFrame({'Label': pred})
# Add 'ImageId' column
df1 = pd.concat([pd.Series(range(1,28001), name='ImageId'), 
                              df[['Label']]], axis=1)


df1.to_csv('submission.csv', index=False)