import numpy as np
import pandas as pd
import tensorflow as tf

import math

training_data = pd.read_csv("../input/train.csv")
testing_data = pd.read_csv("../input/test.csv")


class Dataset(object):
    def __init__(self, data):
        self.rows = len(data.values)
        self.images = data.iloc[:, 1:].values
        self.images = self.images.astype(np.float32)
        self.images = np.multiply(self.images, 1.0 / 255.0)
        self.labels = np.array([np.array([int(i == l) for i in range(10)]) for l in data.iloc[:, 0].values])
        self.index_in_epoch = 0
        self.epoch = 0

    def next_batch(self, batch_size):
        start = self.index_in_epoch
        self.index_in_epoch += batch_size
        if self.index_in_epoch > self.rows:
            self.epoch += 1
            perm = np.arange(self.rows)
            np.random.shuffle(perm)
            self.images = self.images[perm]
            self.labels = self.labels[perm]
            # next epoch
            start = 0
            self.index_in_epoch = batch_size
        end = self.index_in_epoch
        return self.images[start:end], self.labels[start:end]

K = 4
L = 8
M = 12
lr = tf.placeholder(tf.float32)
pkeep = tf.placeholder(tf.float32)
W1 = tf.Variable(tf.truncated_normal([5,5,1,K], stddev=0.1))
B1 = tf.Variable(tf.ones([K])/10)
W2 = tf.Variable(tf.truncated_normal([5,5,K,L], stddev=0.1))
B2 = tf.Variable(tf.ones([L])/10)
W3 = tf.Variable(tf.truncated_normal([4,4,L,M], stddev=0.1))
B3 = tf.Variable(tf.ones([M])/10)

N = 200

W4 = tf.Variable(tf.truncated_normal([7*7*M,N], stddev=0.1))
B4 = tf.Variable(tf.ones([N])/10)
W5 = tf.Variable(tf.truncated_normal([N,10], stddev=0.1))
B5 = tf.Variable(tf.ones([10])/10)

x = tf.placeholder(tf.float32, [None, 28, 28, 1])

Y1 = tf.nn.relu(tf.nn.conv2d(x, W1, strides=[1,1,1,1], padding="SAME") + B1)
Y2 = tf.nn.relu(tf.nn.conv2d(Y1, W2, strides=[1,2,2,1], padding="SAME") + B2)
Y3 = tf.nn.relu(tf.nn.conv2d(Y2, W3, strides=[1,2,2,1], padding="SAME") + B3)

YY = tf.reshape(Y3, shape=[-1, 7*7*M])

Y4 = tf.nn.relu(tf.matmul(YY, W4) + B4)
Y = tf.nn.softmax(tf.matmul(Y4,W5) + B5)

y_ = tf.placeholder(tf.float32, [None, 10])

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(Y), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.005).minimize(cross_entropy)

train_data = Dataset(training_data.iloc[0:37000])
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)

for i in range(15000):
    # learning rate decay
    max_learning_rate = 0.003
    min_learning_rate = 0.0001
    decay_speed = 2000.0
    learning_rate = min_learning_rate + (max_learning_rate - min_learning_rate) * math.exp(-i / decay_speed)
    print(str(i) + ", lr:" + str(learning_rate) )
    
    batch_xs, batch_ys = train_data.next_batch(100)
    batch_xs = np.reshape(batch_xs, [100,28,28,1])

    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys, lr: learning_rate, pkeep: 0.75})

correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

validate_data = Dataset(training_data.iloc[37000:])

print(sess.run(accuracy, feed_dict={x: np.reshape(validate_data.images, [5000, 28, 28, 1]), y_: validate_data.labels}))