import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils
from keras.optimizers import SGD

def getmodel(firsthidden, secondhidden, dropfirst = 0.5, dropsecond = 0.2):
    # Create model
    model = Sequential()
    model.add(Dense(firsthidden, input_dim=Xtrain.shape[1], init='uniform', 
                    activation='relu'))
    # Dropout consists in randomly setting a fraction p of input units to 0 at each update
    # during training time, which helps prevent overfitting
    model.add(Dropout(dropfirst))
    model.add(Dense(secondhidden, init='uniform', activation='relu'))
    model.add(Dropout(dropsecond))
    model.add(Dense(10, init='uniform', activation='sigmoid'))
    # Compile model for a multi-class classification problem
    model.compile(optimizer='rmsprop',
                    loss='categorical_crossentropy', metrics=['accuracy'])
    return model

# The competition datafiles are in the directory ../input
# Read competition data files:
train = pd.read_csv("../input/train.csv")
test  = pd.read_csv("../input/test.csv")

# Write to the log:
print("Training set has {0[0]} rows and {0[1]} columns".format(train.shape))
print("Test set has {0[0]} rows and {0[1]} columns".format(test.shape))
# Any files you write to the current directory get shown as outputs

X = train.values[:, 1:]
n = X.shape[1] # number of feaures
print('Number of features: %d' % n)
scaler = MinMaxScaler()
X = scaler.fit_transform(X)
y = train.values[:, 0]

comp = 400
print('\nPCA analysis: %d components...' % comp)
pca = PCA(n_components = comp)
X = pca.fit_transform(X)
varRatio = pca.explained_variance_ratio_
print('Retained variance: %0.5f' % np.sum(varRatio))

# Create random train and validation sets out of 20% samples
Xtrain, Xval, ytrain, yval = train_test_split(X, y, test_size=0.2,
                                            stratify=y, random_state=11)
print('\nXtrain, ytrain shapes ' + str((Xtrain.shape, ytrain.shape)))
print('Xval, yval shapes ' + str((Xval.shape, yval.shape)))
# number of training examples and features 
#m, n = X.shape

# Basic NN with Keras
# 400, 0.5, 400, 0.2: 97.8
print('\nValidating Keras Neural Network...')
ytrain = np_utils.to_categorical(ytrain, 10) # One hot encoding
yval = np_utils.to_categorical(yval, 10)
firsthidden = [400, 200] #, 800, 1600] #[50, 100, 200, 400]
secondhidden =  [400, 200] #[800, 1600] #☻ [50, 100, 200, 400] 
dropfirst = [0.5]
dropsecond = [0.2]
results = []

for fhid in firsthidden:
    for shid in secondhidden:
        model = getmodel(fhid, shid)
        model.fit(Xtrain, ytrain, nb_epoch=10, batch_size=50, verbose=0)
        trainacc = 100*model.evaluate(Xtrain, ytrain)[1]
        valacc = 100*model.evaluate(Xval, yval)[1]
        results.append([fhid, shid, trainacc, valacc])
        
print('\tfirst hidden \tsecond hidden \ttrain acc \tval acc')
for r in results:
    print('\t%d \t%d \t%0.2f \t%0.2f'% (r[0],r[1],r[2],r[3]))

#Test set
Xtest = test.values
Xtest = scaler.transform(Xtest)
Xtest = pca.transform(Xtest)
print('Test shape: ' + str(Xtest.shape))
pred = np.argmax(model.predict(Xtest), axis=1)
print('pred ' + str(pred[:5]))
# Saving submission file
print('\nSaving submission file...')
pred_digits = np.vstack((np.arange(1, pred.size+1), pred)).T
print('pred digits' + str(pred_digits[:5]))
np.savetxt('DigitRecogSubmissionKerasNN.csv', pred_digits, fmt='%d,%d', delimiter=',', 
        header='ImageId,Label', comments = '')