{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"import tensorflow as tf\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"dtrain  =  pd.read_csv('../input/train.csv')\ndtest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e15b83edfd5c20eeb3f63792d0dd492774024d65","collapsed":true},"cell_type":"code","source":"Xfull, yfull = dtrain.drop('label',axis =1), dtrain['label']\nXtrain, Xvalid, ytrain, yvalid = train_test_split(Xfull, yfull, stratify = yfull, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55575b7362bee29322b85ef141bb4af1d28bb465","collapsed":true},"cell_type":"code","source":"ytrain = pd.get_dummies(ytrain)\nyvalid = pd.get_dummies(yvalid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef88ba76a5aaf9b22f3232c46f131b0edd924bab","collapsed":true},"cell_type":"code","source":"Xtrain = Xtrain.values\nXvalid = Xvalid.values\nytrain = ytrain.values\nyvalid = yvalid.values\nXtest = dtest.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03be6ed532a99cb7a2e50ec135e92a7578fa81e6"},"cell_type":"markdown","source":"## Helper function"},{"metadata":{"trusted":true,"_uuid":"df4faf739822362ff83fa217e6da7864255422bd","collapsed":true},"cell_type":"code","source":"def init_weights(shape):\n    init_random_dist = tf.truncated_normal(shape, stddev=0.1)\n    return tf.Variable(init_random_dist)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a441bdf119be8bf1e96b077ee94709019c22f91","collapsed":true},"cell_type":"code","source":"def init_bias(shape):\n    init_bias_vals = tf.constant(0.1, shape=shape)\n    return tf.Variable(init_bias_vals)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8c8c12da8fcf5aabd39fc3725faf0383c0ee629","collapsed":true},"cell_type":"code","source":"def conv2d(x, W):\n    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3e10804651e791ef668b0a057b414e3e6fa72bb","collapsed":true},"cell_type":"code","source":"def max_pool_2by2(x):\n    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n                          strides=[1, 2, 2, 1], padding='SAME')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13e4abc4249b0263d13bd7285bb133ba58402138","collapsed":true},"cell_type":"code","source":"def convolutional_layer(input_x, shape):\n    W = init_weights(shape)\n    b = init_bias([shape[3]])\n    return tf.nn.relu(conv2d(input_x, W) + b)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f38e1b04c09a54557315ba9751a1b33f7420414","collapsed":true},"cell_type":"code","source":"def normal_full_layer(input_layer, size):\n    input_size = int(input_layer.get_shape()[1])\n    W = init_weights([input_size, size])\n    b = init_bias([size])\n    return tf.matmul(input_layer, W) + b","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cb3a0c2fa7601c6f8a132d155f5c0ea0a34de34","collapsed":true},"cell_type":"code","source":"def get_next_batch(features, labels, j, batch_size):\n    j = j%756\n    return features[j*batch_size:j*batch_size + batch_size, :], labels[j*batch_size:j*batch_size + batch_size, :]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9de77e4b52817cf47c8a972ddbfe4cb501ff7f4a"},"cell_type":"markdown","source":"## Placeholders"},{"metadata":{"trusted":true,"_uuid":"041f8c57149808918976d57c918f50768b9ab29d","collapsed":true},"cell_type":"code","source":"x = tf.placeholder(tf.float32,shape=[None,784])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b3142feb3256e72ec487aff2036e84a8e57f3a0","collapsed":true},"cell_type":"code","source":"y_true = tf.placeholder(tf.float32,shape=[None,10])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43abde06c28d61a71e62771d6a46e46cebd6ae78"},"cell_type":"markdown","source":"## Layers"},{"metadata":{"trusted":true,"_uuid":"ab934e67569a90edec2e8969d4a83b0a33727254","collapsed":true},"cell_type":"code","source":"x_image = tf.reshape(x,[-1,28,28,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ef2720e3e7b309921d7a9840db9dc2c8cf25a20","collapsed":true},"cell_type":"code","source":"# Convolutional Layer 1 and Pool Layer 1\nconvo_1 = convolutional_layer(x_image,shape=[5,5,1,32])\nconvo_1_pooling = max_pool_2by2(convo_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07f761a5c572abce84d7eefea8814ec5f220f247","collapsed":true},"cell_type":"code","source":"# Convolutional Layer 2 and Pool Layer 2\nconvo_2 = convolutional_layer(convo_1_pooling,shape=[5,5,32,64])\nconvo_2_pooling = max_pool_2by2(convo_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e289203b7ac9e2a275d0809b7944d190ee72735","collapsed":true},"cell_type":"code","source":"# Flatten\nconvo_2_flat = tf.reshape(convo_2_pooling,[-1,7*7*64])\n# Dense 1\nfull_layer_one = tf.nn.relu(normal_full_layer(convo_2_flat,1024))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe5a2be2044461d30ee1e94545764e3390077783","collapsed":true},"cell_type":"code","source":"# Dropout\nhold_prob = tf.placeholder(tf.float32)\nfull_one_dropout = tf.nn.dropout(full_layer_one,keep_prob=hold_prob)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99f1978cf04776903f3d99a82ba5e3eb8f8af014","collapsed":true},"cell_type":"code","source":"y_pred = normal_full_layer(full_one_dropout,10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7c1244b8b03f341e5fab50b260dca3f45080b4d"},"cell_type":"markdown","source":"## Loss function"},{"metadata":{"trusted":true,"_uuid":"40834053956ae7a00f20186bd8a87707aeaaa56c","collapsed":true},"cell_type":"code","source":"cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_true,logits=y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3eb478615195f544547534a5390d5a67a725907"},"cell_type":"markdown","source":"## Optimizer"},{"metadata":{"trusted":true,"_uuid":"6afe58caff7aa564d2b1fde2cd96aeaffbbc6dd1","collapsed":true},"cell_type":"code","source":"optimizer = tf.train.AdamOptimizer(learning_rate=0.0001)\ntrain = optimizer.minimize(cross_entropy)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c104cdcb5d47428f1aedcd4229119fc1f4167283"},"cell_type":"markdown","source":"## Initialize Variables"},{"metadata":{"trusted":true,"_uuid":"b763354fc1af53040046ca78391c63b120eb8ccd","collapsed":true},"cell_type":"code","source":"init = tf.global_variables_initializer()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24f39d6759ddea2b914cd6b09f26221d484c7fa9"},"cell_type":"markdown","source":"## Session"},{"metadata":{"trusted":true,"_uuid":"f55eb6d49bcb77d14e8f12b6873b549fc3f2eb1e","collapsed":true},"cell_type":"code","source":"saver = tf.train.Saver()\nsteps = 7560\n\nwith tf.Session() as sess:\n    \n    sess.run(init)\n    \n    for i in range(steps):\n        \n        batch_x , batch_y = get_next_batch(Xtrain, ytrain, i, 50)\n        \n        sess.run(train,feed_dict={x:batch_x,y_true:batch_y,hold_prob:0.5})\n        \n        # PRINT OUT A MESSAGE EVERY 100 STEPS\n        if i%100 == 0:\n            \n            print('Currently on step {}'.format(i))\n            print('Accuracy is:')\n            # Test the Train Model\n            matches = tf.equal(tf.argmax(y_pred,1),tf.argmax(y_true,1))\n\n            acc = tf.reduce_mean(tf.cast(matches,tf.float32))\n\n            print(sess.run(acc,feed_dict={x:Xvalid,y_true:yvalid,hold_prob:1.0}))\n            print('\\n')\n            \n    result = sess.run(y_pred,feed_dict={x:Xtest,hold_prob:0.5})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8af3e4809e10171d1bbe16ab060a6c6b2bac136b","collapsed":true},"cell_type":"code","source":"submission = pd.DataFrame(np.arange(1,28001), columns = ['ImageId'])\nsubmission['Label'] = result.argmax(axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b955b646302732c3de881ad3ba92d89ce235ab27","collapsed":true},"cell_type":"code","source":"submission.to_csv(\"evaluation_submission.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}