# This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load in import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) import tensorflow as tf from sklearn.preprocessing import label_binarize from matplotlib import pyplot as plt from sklearn.utils import shuffle # shuffle the data import keras from keras.models import Sequential from keras.layers import Dense, Dropout, Flatten from keras.layers import Conv2D, MaxPooling2D from keras.utils import np_utils # Input data files are available in the "../input/" directory. # For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory import os print(os.listdir("../input")) # Any results you write to the current directory are saved as output. train = pd.read_csv('../input/train.csv') test = pd.read_csv('../input/test.csv') batch_size = 128 num_classes = 10 epochs = 12 input_shape = ( 28, 28, 1) x_train = train.drop(['label'], axis=1).values.reshape(42000, 28, 28, 1).astype(np.float32)/255 x_test = test.values.reshape(28000, 28, 28, 1).astype(np.float32)/255 y_train = np_utils.to_categorical( train['label'].values, num_classes) print('x_train shape:', x_train.shape) print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') model = Sequential() model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1) y_test = model.predict(x_test) y_pred = [np.argmax(Y) for Y in y_test] output = pd.DataFrame({'ImageId':pd.Series(np.arange(1,len(x_test)+1)), 'Label': pd.Series(y_pred) }) output.to_csv('predictions.csv', index=False) ''' RESHAPED = 784 NB_CLASSES = 10 NB_EPOCH = 200 BATCH_SIZE = 128 VERBOSE = 1 OPTIMIZER = SGD() N_HIDDEN = 128 VALIDATION_SPLIT = 0.2 DROPOUT = 0.3 #M_HIDDEN hidden layers 10 outputs model = Sequential() model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,))) model.add(Activation('relu')) model.add(Dropout(DROPOUT)) model.add(Dense(N_HIDDEN)) model.add(Activation('relu')) model.add(Dropout(DROPOUT)) model.add(Dense(N_HIDDEN)) model.add(Activation('relu')) model.add(Dropout(DROPOUT)) model.add(Dense(NB_CLASSES)) model.add(Activation('softmax')) model.summary() model.compile(loss = 'categorical_crossentropy', optimizer=OPTIMIZER, metrics = ['accuracy']) history = model.fit(X_train, Y_train, batch_size = BATCH_SIZE, epochs = NB_EPOCH, verbose = VERBOSE, validation_split = VALIDATION_SPLIT) Y_test = model.predict(X_test) y_pred = [np.argmax(Y) for Y in Y_test] output = pd.DataFrame({'ImageId':pd.Series(np.arange(1,len(X_test)+1)), 'Label': pd.Series(y_pred) }) output.to_csv('predictions.csv', index=False) THE FOLLOWING IS USING CNN (LENET-5):W from tensorflow.contrib.layers import flatten X_train = train.drop(['label'], axis = 1).values.reshape(-1,28,28,1) X_test = test.values.reshape(-1,28,28,1) y_train = train['label'].values #y_train = label_binarize(y_train.values, classes=[0,1,2,3,4,5,6,7,8,9]) # Pad images with 0s X_train = np.pad(X_train, ((0,0),(2,2),(2,2),(0,0)), 'constant') X_test = np.pad(X_test, ((0,0),(2,2),(2,2),(0,0)), 'constant') print("Updated Image Shape: {}".format(X_train[0].shape)) index = np.random.randint(0, len(X_train)) image = X_train[index].squeeze() plt.figure(figsize=(1,1)) plt.imshow(image, cmap="gray") print(y_train[index]) # shuffle the data X_train, y_train = shuffle(X_train, y_train) EPOCHS = 10 BATCH_SIZE = 128 def LeNet(x): # Hyperparameters mu = 0 sigma = 0.1 layer_depth = { 'layer_1' : 6, 'layer_2' : 16, 'layer_3' : 120, 'layer_f1' : 84 } # Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6. conv1_w = tf.Variable(tf.truncated_normal(shape = [5,5,1,6],mean = mu, stddev = sigma)) conv1_b = tf.Variable(tf.zeros(6)) conv1 = tf.nn.conv2d(x,conv1_w, strides = [1,1,1,1], padding = 'VALID') + conv1_b # Activation. conv1 = tf.nn.relu(conv1) # Pooling. Input = 28x28x6. Output = 14x14x6. pool_1 = tf.nn.max_pool(conv1,ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID') # Layer 2: Convolutional. Output = 10x10x16. conv2_w = tf.Variable(tf.truncated_normal(shape = [5,5,6,16], mean = mu, stddev = sigma)) conv2_b = tf.Variable(tf.zeros(16)) conv2 = tf.nn.conv2d(pool_1, conv2_w, strides = [1,1,1,1], padding = 'VALID') + conv2_b # Activation. conv2 = tf.nn.relu(conv2) # Pooling. Input = 10x10x16. Output = 5x5x16. pool_2 = tf.nn.max_pool(conv2, ksize = [1,2,2,1], strides = [1,2,2,1], padding = 'VALID') # Flatten. Input = 5x5x16. Output = 400. fc1 = flatten(pool_2) # Layer 3: Fully Connected. Input = 400. Output = 120. fc1_w = tf.Variable(tf.truncated_normal(shape = (400,120), mean = mu, stddev = sigma)) fc1_b = tf.Variable(tf.zeros(120)) fc1 = tf.matmul(fc1,fc1_w) + fc1_b # Activation. fc1 = tf.nn.relu(fc1) # Layer 4: Fully Connected. Input = 120. Output = 84. fc2_w = tf.Variable(tf.truncated_normal(shape = (120,84), mean = mu, stddev = sigma)) fc2_b = tf.Variable(tf.zeros(84)) fc2 = tf.matmul(fc1,fc2_w) + fc2_b # Activation. fc2 = tf.nn.relu(fc2) # Layer 5: Fully Connected. Input = 84. Output = 10. fc3_w = tf.Variable(tf.truncated_normal(shape = (84,10), mean = mu , stddev = sigma)) fc3_b = tf.Variable(tf.zeros(10)) logits = tf.matmul(fc2, fc3_w) + fc3_b return logits x = tf.placeholder(tf.float32, (None, 32, 32, 1)) y = tf.placeholder(tf.int32, (None)) one_hot_y = tf.one_hot(y, 10) rate = 0.001 logits = LeNet(x) cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits = logits, labels = one_hot_y) loss_operation = tf.reduce_mean(cross_entropy) optimizer = tf.train.AdamOptimizer(learning_rate = rate) training_operation = optimizer.minimize(loss_operation) #model evaluation correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1)) accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) def evaluate(X_data, y_data): num_examples = len(X_data) total_accuracy = 0 sess = tf.get_default_session() for offset in range(0, num_examples, BATCH_SIZE): batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE] accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y}) total_accuracy += (accuracy * len(batch_x)) return total_accuracy / num_examples with tf.Session() as sess: sess.run(tf.global_variables_initializer()) num_examples = len(X_train) print("Training...") for i in range(EPOCHS): X_train, y_train = shuffle(X_train, y_train) X_validation = X_train[num_examples-BATCH_SIZE:] y_validation = y_train[num_examples-BATCH_SIZE:] for offset in range(0, num_examples-BATCH_SIZE, BATCH_SIZE): end = offset + BATCH_SIZE batch_x, batch_y = X_train[offset:end], y_train[offset:end] sess.run(training_operation, feed_dict={x: batch_x, y: batch_y}) validation_accuracy = evaluate(X_validation, y_validation) print("EPOCH {} ...".format(i+1)) print("Validation Accuracy = {:.3f}".format(validation_accuracy)) print("Predicting:") y_pred = sess.run(tf.argmax(logits,1), feed_dict={x:X_test}) output = pd.DataFrame({'ImageId':pd.Series(np.arange(1,len(X_test)+1)), 'Label': pd.Series(y_pred) }) output.to_csv('predictions.csv', index=False) '''