{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"train_data = pd.read_csv(\"../input/train.csv\")\ntest_data = pd.read_csv(\"../input/test.csv\")","execution_count":2,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fa6350144d2f82357a44065763cf9f172f46715"},"cell_type":"code","source":"print(test_data.columns)","execution_count":3,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"866bda6384d97ec5d69353412f31ec93ba9272b3","collapsed":true},"cell_type":"code","source":"label_col = ['label']\n\ntrain_x = train_data.drop(columns=label_col)\ntrain_y = train_data[label_col]\n\ntest_x = test_data[:]","execution_count":4,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"12c239706297310fda156d4a620266d020eaeb1d"},"cell_type":"code","source":"from sklearn import model_selection\n\ntrain_x, valid_x, train_y, valid_y = model_selection.train_test_split(train_x, train_y)","execution_count":5,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deb8039ee02c2940dbd5c9159479d293e8653f52"},"cell_type":"code","source":"print(train_x.shape)\nprint(train_y.shape)\n\nprint('Valid X', valid_x.shape)\nprint('Valid Y', valid_y.shape)","execution_count":6,"outputs":[]},{"metadata":{"_uuid":"ebb5372e5f4ef41b2e2f06ce41f5f720e0b68c89"},"cell_type":"markdown","source":"## Transform to np.array"},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"fc825265307dcdc47f7d1d552ad8cc677e10f135"},"cell_type":"code","source":"import numpy as np","execution_count":7,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d49b220087ac32265dca457d63241fae149f11b","collapsed":true},"cell_type":"code","source":"train_x_np = np.reshape(np.array(train_x), [len(train_x), 28 * 28])\nvalid_x_np = np.reshape(np.array(valid_x), [len(valid_x), 28 * 28])\n\ntest_x_np = np.array(test_x).reshape([len(test_x), 28 * 28])","execution_count":8,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adbb744240e97a6c40e6cb421b0be57e56608bd9"},"cell_type":"code","source":"print(train_x_np.shape)\nprint(valid_x_np.shape)\n\nprint(test_x_np.shape)","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"4308e1ca7160d8cf70c0e81494948fe3ea81514d"},"cell_type":"markdown","source":"## Create input for labels"},{"metadata":{"trusted":true,"_uuid":"0703a6b90a766d332f41744d8cb79f883c31d7e1","collapsed":true},"cell_type":"code","source":"n_classes = 10\n\ntrain_y_np = np.zeros([len(train_y), n_classes])\nvalid_y_np = np.zeros([len(valid_y), n_classes])\n\nt_y = np.array(train_y).reshape([len(train_y)])\nv_y = np.array(valid_y).reshape([len(valid_y)])\n\n#print(train_y)\nfor i in range(len(train_y)):\n    idx = t_y[i]\n    train_y_np[i,idx] = 1\n    \n\n#print(train_y)\nfor i in range(len(valid_y)):\n    idx = v_y[i]\n    valid_y_np[i,idx] = 1","execution_count":10,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc47d7648f1f535decc26734e378d30dc89baf12"},"cell_type":"code","source":"print(train_y_np.shape)\nprint(valid_y_np.shape)","execution_count":11,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0af16869e12d6682d04ef5a313b355c6a3974b0"},"cell_type":"code","source":"import tensorflow as tf","execution_count":12,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"804a3a108a97b468e23b4bdef787663083ec8d56"},"cell_type":"code","source":"learning_rate = 0.001\ndropout_rate = 0.8","execution_count":13,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d12e7a6439883ea4417529a650b792e044693649"},"cell_type":"code","source":"def extract_batch(x, y, batch_size, curr):\n    batch_x = np.zeros([batch_size, x.shape[1]])\n    batch_y = np.zeros([batch_size, y.shape[1]])\n    \n    for i in range(batch_size):\n        batch_x[i] = x[curr]\n        batch_y[i] = y[curr]\n        curr += 1\n        if (curr >= x.shape[0]):\n            curr = 0\n            \n    return batch_x, batch_y, curr","execution_count":14,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3620ffda9fb71e6c5fe053f633407a8a2970569","collapsed":true},"cell_type":"code","source":"def create_model(x, y):\n    x_resh = tf.reshape(x, shape=[-1, 28, 28, 1])\n    \n    w_0 = tf.Variable(tf.random_normal([3, 3, 1, 16]))\n    b_0 = tf.Variable(tf.random_normal([16]))\n    \n    conv_0 = tf.nn.conv2d(input=x_resh, filter=w_0, strides=[1, 1, 1, 1], padding='SAME')\n    conv_0 = tf.nn.bias_add(conv_0, b_0)\n    \n    pool_0 = tf.nn.max_pool(value=conv_0, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n    \n    w_1 = tf.Variable(tf.random_normal([3, 3, 16, 32]))\n    b_1 = tf.Variable(tf.random_normal([32]))\n    \n    conv_1 = tf.nn.conv2d(input=pool_0, filter=w_1, strides=[1, 1, 1, 1], padding='SAME')\n    conv_1 = tf.nn.bias_add(conv_1, b_1)\n    \n    pool_1 = tf.nn.max_pool(value=conv_1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n    \n    drop_1 = tf.nn.dropout(pool_1, dropout_rate)\n    \n    w_2 = tf.Variable(tf.random_normal([3, 3, 32, 64]))\n    b_2 = tf.Variable(tf.random_normal([64]))\n    \n    conv_2 = tf.nn.conv2d(input=drop_1, filter=w_2, strides=[1, 1, 1, 1], padding='SAME')\n    conv_2 = tf.nn.bias_add(conv_2, b_2)\n    \n    drop_2 = tf.nn.dropout(conv_2, dropout_rate)\n    #pool_2 = tf.nn.max_pool(value=conv_2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n    \n    w_fc = tf.Variable(tf.random_normal([7 * 7 * 64, 1024]))\n    b_fc = tf.Variable(tf.random_normal([1024]))\n    \n    flat_0 = tf.reshape(drop_2, [-1, 7 * 7 * 64])\n    \n    fc_0 = tf.nn.xw_plus_b(flat_0, w_fc, b_fc)\n    fc_0 = tf.nn.relu(fc_0)\n    \n    w_out = tf.Variable(tf.random_normal([1024, n_classes]), name='w_out')\n    b_out = tf.Variable(tf.random_normal([n_classes]), name='b_out')\n    \n    out = tf.nn.xw_plus_b(fc_0, w_out, b_out)\n    return out","execution_count":15,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ce62031f6ba57c65583ad8018dac23efd1fed4a"},"cell_type":"code","source":"x = tf.placeholder(shape=[None, 28 * 28], dtype=tf.float32)\ny = tf.placeholder(shape=[None, 10], dtype=tf.float32)\n\nlogits = create_model(x, y)\n\npreds = tf.nn.softmax(logits=logits)\n\nloss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))\n\noptimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\nopt_op = optimizer.minimize(loss_op)\n\ncorrect_pred = tf.equal(x=tf.argmax(preds, 1), y=tf.argmax(y, 1))\nacc_op = tf.reduce_mean(tf.cast(x=correct_pred, dtype=tf.float32))\n\ninit = tf.global_variables_initializer()","execution_count":16,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ebd3830771ed058599b019eab597bbd3ab1f74d"},"cell_type":"code","source":"n_steps = 6000\nbatch_size = 200\n\nsess = tf.Session()\n#with tf.Session() as sess:\nsess.run(init)\ncurr = 0\nfor step in range(n_steps):\n    # TODO extract batch\n\n    train_x_batch, train_y_batch, curr = extract_batch(x=train_x_np, y=train_y_np, batch_size=batch_size, curr = curr)\n    feed_dict = {x: train_x_batch, y: train_y_batch}\n\n    sess.run(opt_op, feed_dict=feed_dict)\n\n    if (step % 10 == 0):\n        loss, acc = sess.run([loss_op, acc_op], feed_dict=feed_dict)\n\n        #print(loss)\n        #print(acc)\n\n        print(\"Step: %.2d\" % step, \" Loss: \", \"{:.2f}\".format(loss), \" Accuracy: \", \"{:.2f}\".format(acc))\n\n    # validation\n\nprint(\"Optimization finished\")\n\nloss, acc = sess.run([loss_op, acc_op], feed_dict={x: valid_x_np, y: valid_y_np})\n\nprint(\"Validation Loss: \", \"{:.2f}\".format(loss), \" Accuracy: \", \"{:.2f}\".format(acc))","execution_count":17,"outputs":[]},{"metadata":{"_uuid":"a200fcfc000bd72592e988ed9d4b78d0b48b3dae"},"cell_type":"markdown","source":"## Classify test data"},{"metadata":{"trusted":true,"_uuid":"d7ebe501b80392e03c46b17a22f81c9b3f6b2bf3","collapsed":true},"cell_type":"code","source":"predictions = sess.run(preds, feed_dict={x: test_x})","execution_count":120,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"240ddc98974c9f8dd5360bf57eeae07830c4ed80","collapsed":true},"cell_type":"code","source":"result = np.zeros([len(predictions)], dtype=np.int64)\n\nfor idx, pred in enumerate(predictions):\n    # pred = 1d array\n    result[idx] = np.argmax(pred)","execution_count":131,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e900de4c18a094af6e6aaf724cea05177e8f42c","collapsed":true},"cell_type":"code","source":"df = pd.DataFrame(data={'ImageId': range(1,len(predictions)+1), 'Label': result})\ndf.to_csv('result.csv', index=False)","execution_count":139,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}