{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"9d80fa72-e24c-8cd5-aa7e-60e30d566a97"},"source":"Hello, i am new at Machine Learning, i will try my best."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"1703207d-1050-2bee-d39f-22cc8c4bdbce"},"outputs":[],"source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nfrom __future__ import print_function\nfrom subprocess import check_output\nfrom six.moves import cPickle as pickle\nfrom six.moves import range\nfrom distutils.version import LooseVersion as Version\nfrom sklearn import __version__ as sklearn_version\nif Version(sklearn_version) < '0.18':\n    from sklearn.grid_search import train_test_split\nelse:\n    from sklearn.model_selection import train_test_split\n    \nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"},{"cell_type":"markdown","metadata":{"_cell_guid":"ffd8be0c-28f2-74b8-e909-4598f74cd794"},"source":"Data Preparation"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d5be89a6-c32d-c66b-6d4e-0134b847302f"},"outputs":[],"source":"# constants\nimage_size = 28\n\nnum_labels = 10\n\nbatch_size = 128\n\nhidden_n_1 = 1024\nhidden_n_2 = 196\n\nbeta = 0.01\n\nlearning_rate = 0.0001\ntraining_epochs = 100\n\n\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n\ntest_data = train.values.astype(np.float32)\nprint(test_data.shape)\ntrain_data = train.ix[:,1:].values.astype(np.float32)\ntrain_labels = train.ix[:,0].values.astype(np.float32)\n\ntrain_data, test_data, train_labels, test_labels = train_test_split(\n    train_data, train_labels, test_size=0.33, random_state=42\n)\ntest_data, validation_data, test_labels, validation_labels = train_test_split(\n    test_data, test_labels, test_size=0.33, random_state=42\n)\n\nprint('Train data shape %s, Train data labels shape %s' % (train_data.shape, train_labels.shape))\nprint('Test data shape %s, Test data labels shape %s' % (test_data.shape,  test_labels.shape))\nprint('Validation data shape %s, Train data labels shape %s' % (validation_data.shape, validation_labels.shape))\n\ndef reformat(dataset, labels):\n    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n    return dataset, labels\n\ntrain_dataset, train_labels = reformat(train_data, train_labels)\nvalid_dataset, validation_labels = reformat(validation_data, validation_labels)\ntest_dataset, test_l = reformat(test_data, test_labels)\n\nprint('Train data shape {}'.format(train_dataset.shape))\nprint('Train data labels shape {}'.format(train_labels.shape))\n\nprint('Test data shape {}'.format(test_dataset.shape))\nprint('Test data labels shape {}'.format(test_labels.shape))\n\nprint('Validation data shape {}'.format(valid_dataset.shape))\nprint('Validation data labels shape {}'.format(validation_labels.shape))"},{"cell_type":"markdown","metadata":{"_cell_guid":"80f84a38-560a-9917-7eb4-fd3e8de6f63d"},"source":"\n\nDeclaring Variables\n-------------------------------------------------------------------------------------------------------------------------------\n"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"8e938a9c-34c0-fbc9-cf0e-71f864f7739c"},"outputs":[],"source":"# helper functions\ndef weight_variable(shape):\n    initial = tf.truncated_normal(shape, stddev=0.01)\n    return tf.Variable(tf.truncated_normal(shape, stddev=0.01))\n\ndef bias_variable(shape):\n    initial = tf.constant(0.1, shape=shape)\n    return tf.Variable(tf.constant(0.1, shape=shape))\n\n\nwith tf.name_scope('constants'):\n    ValDataset = tf.constant(valid_dataset)\n    TestDataset = tf.constant(test_dataset)\n\n# input images\nwith tf.name_scope('train_input'):\n    train = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size), name=\"x-input-train\")\n    # target 10 output classes\n    labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels), name=\"y-input-train\")\n\n# model parameters will change during training so we use tf.Variable\nwith tf.name_scope(\"weights\"):\n    W1 = weight_variable([image_size * image_size, hidden_n_1]) # in [784, 10]\n    W2 = tf.Variable(tf.random_normal([hidden_n_1, hidden_n_2]))   # hidden [1024, 196]\n    W3 = tf.Variable(tf.random_normal([hidden_n_2, num_labels]))  # out [196, 10]\n\n\nwith tf.name_scope('biases'):\n    B1 = bias_variable([hidden_n_1]) # [1024]\n    B2 = tf.Variable(tf.zeros([hidden_n_2]))  # [196]\n    B3 = tf.Variable(tf.zeros([num_labels]))  # [10]\n\nwith tf.name_scope('logistic'):\n    L1 = tf.matmul(train, W1) + B1\n    Relu1 = tf.nn.relu(L1)\n    L2 = tf.matmul(Relu1, W2) + B2\n    Relu2 = tf.nn.relu(L2)\n    L3 = tf.matmul(Relu2, W3) + B3\n\n# implement model\nwith tf.name_scope('softmax'):\n    # y is our prediction\n    prediction = tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=L3)  # softmax\n\nwith tf.name_scope('loss'):\n    loss = tf.reduce_mean(prediction)\n\nwith tf.name_scope('regularization_L2'):\n    regularizer = tf.nn.l2_loss(W3)\n    loss = tf.reduce_mean(loss + beta * regularizer)\n\n# specify optimizer\nwith tf.name_scope('train'):\n    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)\n\nwith tf.name_scope('train_prediction'):\n    train_prediction = tf.nn.softmax(L3)\n\nwith tf.name_scope('valid_prediction'):\n    # Predictions for validation\n    valid_1 = tf.nn.relu(tf.matmul(ValDataset, W1) + B1)\n    valid_2 = tf.nn.relu(tf.matmul(valid_1, W2) + B2)\n    valid_3 = tf.matmul(valid_2, W3) + B3\n    valid_prediction = tf.nn.softmax(valid_3)\n\nwith tf.name_scope('test_prediction'):\n    # Predictions for test\n    test_1 = tf.nn.relu(tf.matmul(TestDataset, W1) + B1)\n    test_2 = tf.nn.relu(tf.matmul(test_1, W2) + B2)\n    test_3 = tf.matmul(test_2, W3) + B3\n    test_prediction = tf.nn.softmax(test_3)\n\nwith tf.name_scope('Minibatch_accuracy'):\n    minibatch_correct_prediction = tf.equal(tf.argmax(train_prediction, 1), tf.argmax(labels, 1))\n    minibatch_accuracy = 100.0 * tf.reduce_mean(tf.cast(minibatch_correct_prediction, tf.float32))\n\nwith tf.name_scope('Validation_accuracy'):\n    valid_correct_prediction = tf.equal(tf.argmax(valid_prediction, 1), tf.argmax(validation_labels, 1))\n    valid_accuracy = 100.0 * tf.reduce_mean(tf.cast(valid_correct_prediction, tf.float32))\n\nwith tf.name_scope('Test_accuracy'):\n    test_correct_prediction = tf.equal(tf.argmax(test_prediction, 1), tf.argmax(test_l, 1))\n    test_accuracy = 100.0 * tf.reduce_mean(tf.cast(test_correct_prediction, tf.float32))"},{"cell_type":"markdown","metadata":{"_cell_guid":"a301694e-abfb-6764-ecc4-e83577391038"},"source":"## Session ##"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"db87fbdb-bd79-2c45-e320-e2794f7c9bf1"},"outputs":[],"source":"with tf.Session() as sess:\n    # variables need to be initialized before we can use them\n    tf.initialize_all_variables().run()\n    print(\"Initialized\")\n    batch_count = int(train_dataset.shape[0] / batch_size)\n    for epoch in range(training_epochs):\n        for step in range(batch_count):\n            offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n            batch_data = train_dataset[offset:(offset + batch_size), :]\n            batch_labels = train_labels[offset:(offset + batch_size), :]\n            batch_feed_dict = {train: batch_data, labels: batch_labels}\n            _, l, predictions = sess.run([optimizer, loss, train_prediction], feed_dict=batch_feed_dict)\n            if (step % 5 == 0):\n                print(\"Minibatch loss at step %d: %f\" % (step, l))\n                minibatch_acc = sess.run([test_prediction,\n                                          minibatch_correct_prediction,\n                                          minibatch_accuracy], feed_dict=batch_feed_dict)\n                print(\"Minibatch accuracy: %.1f%%\" % minibatch_acc[-1])\n                val_acc = sess.run([valid_accuracy])\n                print(\"Validation accuracy: %.1f%%\" % val_acc[-1])\n        test_acc = sess.run([test_accuracy])\n        print(\"Test accuracy: %.1f%%\" % test_acc[-1])"}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.0"}},"nbformat":4,"nbformat_minor":0}