{"nbformat": 4, "metadata": {"kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}, "language_info": {"mimetype": "text/x-python", "file_extension": ".py", "nbconvert_exporter": "python", "version": "3.6.3", "name": "python", "pygments_lexer": "ipython3", "codemirror_mode": {"name": "ipython", "version": 3}}}, "nbformat_minor": 1, "cells": [{"outputs": [], "execution_count": null, "cell_type": "code", "source": ["# This Python 3 environment comes with many helpful analytics libraries installed\n", "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n", "# For example, here's several helpful packages to load in \n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "\n", "# Input data files are available in the \"../input/\" directory.\n", "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output."], "metadata": {"_cell_guid": "d2036ec6-4f2a-459f-b468-a16dd3747be2", "_uuid": "06b5a6116c079e6bd56f6914d5768f7c74e3127f"}}, {"cell_type": "markdown", "source": ["**Import Data**"], "metadata": {"_cell_guid": "84b28248-ad87-482e-97a3-a89e30e30d35", "_uuid": "d7a20ca4ce23671892ade8833f8140647291e049"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["df = pd.read_csv( '../input/train.csv' )\n", "data = df.as_matrix()"], "metadata": {"collapsed": true, "_cell_guid": "8604221a-bc15-4960-ad98-e0a2c6969e3e", "_uuid": "572ceb301e900a3eb00adffaeff543388fd1055f"}}, {"cell_type": "markdown", "source": ["**Train / Test Split**"], "metadata": {"_cell_guid": "e4a7790a-cc59-47fb-bb4a-e54707bce556", "_uuid": "c28736ee0903e7700a639c30414cd0ceb99dcc4d"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["from sklearn.model_selection import train_test_split\n", "\n", "train_x, test_x, train_y, test_y = train_test_split( data[:,1:].astype( 'float32' ), \n", "                                                    data[:,0].astype( 'float32' ), \n", "                                                    test_size=0.33, \n", "                                                    random_state=4 )"], "metadata": {"collapsed": true, "_cell_guid": "10aad4e4-2e65-4f76-86a2-28efd1fffae7", "_uuid": "f93b142e370ddba4cf386929a6964938fb06d75d"}}, {"cell_type": "markdown", "source": ["**Display Samples**"], "metadata": {"_cell_guid": "25829cd0-3ac2-4d7d-8410-bbd5655b06b5", "_uuid": "5bb0c7eb5fc0fca28a6dbc4c182d7b6253184ae8"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["plt.figure( figsize=( 14,10 ) )\n", "for i in range( 50 ):\n", "    sample = np.random.randint( 0, train_x.shape[0], 1 )[0]\n", "    plt.subplot(5, 10, i+1)\n", "    plt.tick_params( axis='both', which='both', bottom='off', left='off', labelbottom='off', labelleft='off' )\n", "    plt.imshow( train_x[sample].reshape((28, 28)), cmap='gray' )\n", "    plt.title( int( train_y[sample] ) )"], "metadata": {"_cell_guid": "f5501c86-e038-4603-bd33-f05f9a63b7d8", "_uuid": "a17cead5c42d5676afe9631154e561f145313563"}}, {"cell_type": "markdown", "source": ["**Prepare Data**"], "metadata": {"_cell_guid": "c70ca03a-afa4-4fb6-b184-beeb9c96f345", "_uuid": "3cb5fc872726908bef8ec5300ce7cdbd6de6d483"}}, {"cell_type": "markdown", "source": ["Normalize"], "metadata": {"_cell_guid": "4383e4b4-6ca4-4078-b973-99b42fd26676", "_uuid": "92439bfe6710c81aaaf9a626f9d7765a3ba5ae5b"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["print(\"Min: %.2f\" % np.min(train_x))\n", "print(\"Max: %.2f\" % np.max(train_x))\n", "train_x /= 255\n", "test_x /= 255\n", "print(\"Min: %.2f\" % np.min(train_x))\n", "print(\"Max: %.2f\" % np.max(train_x))"], "metadata": {"_cell_guid": "b3f3da5c-da39-46c2-8023-8d504474b301", "_uuid": "5e22b0446279c4320e05e52425f4d81b580e4ccf"}}, {"cell_type": "markdown", "source": ["One-hot encoding"], "metadata": {"_cell_guid": "2f441ec4-cc9b-40a6-ac6b-eadabea90ec2", "_uuid": "6a2403c15cbdfcfd98c150be362957bd38702dbe"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["train_y = pd.get_dummies( train_y ).as_matrix()\n", "test_y = pd.get_dummies( test_y ).as_matrix()\n", "train_y"], "metadata": {"_cell_guid": "6aebf0c0-45c5-4fa0-9e22-f904a76de409", "_uuid": "99d7f819f3730fd4e41adacda34912b75d0c0fa8"}}, {"cell_type": "markdown", "source": ["**Create Model**"], "metadata": {"_cell_guid": "0733dc61-e79d-49f6-97e1-1fec3e6e65e3", "_uuid": "fe2c794c8ffdf3c8d3194d84720c7366e1e3c646"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["import tensorflow as tf"], "metadata": {"collapsed": true, "_cell_guid": "ba70b109-abe8-44e6-ba98-19d41bb79fb2", "_uuid": "0591ae29a05bf553da4927054749b66f85cc988c"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["x = tf.placeholder( tf.float32, [None, 784] )\n", "\n", "W = tf.Variable( tf.random_normal( [784, 10], mean=0.1, seed=4 ) )\n", "b = tf.Variable( tf.random_normal( [10], mean=0.1, seed=4 ))\n", "\n", "# dropout = tf.nn.dropout( tf.matmul( x, W ) + b, 0.75 )\n", "y = tf.nn.softmax( tf.matmul( x, W ) + b )"], "metadata": {"collapsed": true, "_cell_guid": "341c953d-c6bf-4365-8dad-31f09f7be036", "_uuid": "b0a074c4373c9c09947a9eaa8e0b7139e6737f90"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["y_ = tf.placeholder( tf.float32, [None, 10] )"], "metadata": {"collapsed": true, "_cell_guid": "f734e456-4739-4865-8f8f-429fe8ce6f9b", "_uuid": "059b65be71cfaafb9319937076f7dfb1d01602d3"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["# loss function\n", "cross_entropy = tf.reduce_mean( -tf.reduce_sum( y_ * tf.log( y ), reduction_indices=[1] ) )"], "metadata": {"collapsed": true, "_cell_guid": "fc9b5959-6e35-45ce-816d-0b3d4726b014", "_uuid": "74aa453a624905301b071316e7c1a0824d5500fb"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["# optimise loss\n", "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n", "train_size = train_x.shape[0]\n", "train_size"], "metadata": {"_kg_hide-input": false, "_cell_guid": "0d6e141f-a60a-4c23-81f9-2bb4664dbcf8", "_uuid": "2d5b756ce072e5b69d75f46c2ba1288cef06401c"}}, {"cell_type": "markdown", "source": ["**Train**"], "metadata": {"_cell_guid": "0fbe55e2-0bbe-4b89-9203-8ce4319a695f", "_uuid": "99c90b94f21b6fa09bb78fa282f707dfb78128c1"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["correct_pred = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"], "metadata": {"collapsed": true, "_cell_guid": "7afca125-d170-4376-be6d-617c54e765be", "_uuid": "aa4d1f6a76e0e405541c59ae5ec23969d1960288"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["# training\n", "train_size = train_x.shape[0]\n", "batch_size = 100\n", "\n", "sess = tf.InteractiveSession()\n", "\n", "tf.global_variables_initializer().run()\n", "for i in range( 0, 500000, batch_size ):\n", "    idx = i%train_size\n", "    sess.run( train_step, feed_dict={x: train_x[idx:idx+batch_size], \n", "                                     y_: train_y[idx:idx+batch_size]} )\n", "    if i%10000==0:\n", "        loss, acc = sess.run( [cross_entropy, accuracy], feed_dict={x: train_x[idx:idx+batch_size], y_: train_y[idx:idx+batch_size]} )\n", "        print( 'Step: ' + str(i) + ' loss: ' + str(loss) + ' accuracy: ' + str(acc) )"], "metadata": {"_cell_guid": "c75417c0-ad32-4a08-b902-472ec830eacf", "_uuid": "5562253177b2726ef31cfac27902f71fb3f80932"}}, {"cell_type": "markdown", "source": ["**Evaluate with unseen data**"], "metadata": {"_cell_guid": "f1446881-11e6-4cad-ba67-75d56c107e4f", "_uuid": "df548870295c768f80aa4a5d31b55a11142b9a9e"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["tested_y = sess.run( tf.argmax( y, 1 ), feed_dict = {x: test_x} )\n", "correct_prediction = np.equal( tested_y, np.argmax( test_y, 1 ) )"], "metadata": {"collapsed": true, "_cell_guid": "50aebb10-d3b6-4046-adf1-da7c4208e7cc", "_uuid": "b1d1acf37b52b537241b4b99c82d05c9efa312e1"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["#correct_prediction = tf.equal( tf.argmax( y,1 ), tf.argmax (y_,1 ) )\n", "p = tf.placeholder(tf.bool, [None,])\n", "accuracy = tf.reduce_mean( tf.cast( p, tf.float32 ) )\n", "\n", "print( 'Accuracy (on unseen data):', sess.run( accuracy, feed_dict={ p: correct_prediction } ) )\n", "print('Correct:',\n", "      sum(correct_prediction.astype(int)), \n", "      ' out of', \n", "      len(test_x) )"], "metadata": {"scrolled": true, "_cell_guid": "14833b6d-64d4-40a8-8e13-e981e504a80b", "_uuid": "494cfb69ec79001375c8215ad1e2df1726cbebec"}}, {"cell_type": "markdown", "source": ["**Failures**"], "metadata": {"_cell_guid": "c91d9846-b14e-4bd5-a384-8a10123fee2f", "_uuid": "596b15f2e5a87008344fab5c76150ba559f55b5b"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["test_combine = np.concatenate( ( np.array( [correct_prediction] ).T,np.array([tested_y]).T,test_x ),axis=1 )\n", "failures = np.array( list( x for x in test_combine if x[0]==0 ) )\n", "print( 'Number of failures: %s'%failures.shape[0] )"], "metadata": {"_cell_guid": "e62ef1ba-824b-4985-acd4-a80c58812d05", "_uuid": "3851078b8d0580cd1a1673801272ebda52a0f167"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["plt.figure( figsize=( 14,10 ) )\n", "for i in range( 50 ):\n", "    sample = np.random.randint( 0, failures.shape[0], 1 )[0]\n", "    plt.subplot(5, 10, i+1)\n", "    plt.tick_params( axis='both', which='both', bottom='off', left='off', labelbottom='off', labelleft='off' )\n", "    plt.imshow( failures[sample,2:].reshape((28, 28)), cmap='gray' )\n", "    plt.title( int( failures[sample, 1] ) )"], "metadata": {"_cell_guid": "0f471714-20cc-424d-9b1a-36c67c94322c", "_uuid": "265b7d75972abb53128d79d475983bc4f0506565"}}, {"cell_type": "markdown", "source": ["**Predict**"], "metadata": {"_cell_guid": "f79e447d-1b80-4300-a699-61a0f670081e", "_uuid": "97298d372c167ad4460eb9f21f9dc398bef8c992"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["df = pd.read_csv( '../input/test.csv' )\n", "predict_x = df.as_matrix() / 255\n", "\n", "predict_y = tf.argmax( y, 1 )\n", "predicted_y = sess.run( predict_y, feed_dict = {x: predict_x} )"], "metadata": {"collapsed": true, "_cell_guid": "6abd0fb0-9ad1-4336-9f68-537a99fa60fa", "_uuid": "17b524722054efebe51f5548a5f5487d4cbc5656"}}, {"cell_type": "markdown", "source": ["**Visualize sample predictions**"], "metadata": {"_cell_guid": "e774c9ff-cfab-49d6-af08-5438f842676f", "_uuid": "e87b8311d7492ae9cf387857b49d456adc656e7a"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["plt.figure( figsize=( 14,10 ) )\n", "for i in range( 50 ):\n", "    sample = np.random.randint( 0, predict_x.shape[0], 1 )[0]\n", "    plt.subplot(5, 10, i+1)\n", "    plt.tick_params( axis='both', which='both', bottom='off', left='off', labelbottom='off', labelleft='off' )\n", "    plt.imshow( predict_x[sample].reshape((28, 28)), cmap='gray' )\n", "    plt.title( predicted_y[sample] )"], "metadata": {"_cell_guid": "300fe95d-cdb0-4ddb-abcd-87a8715c86a6", "_uuid": "a568ad4497da10cc6a690684ea713c959383c5a9"}}, {"cell_type": "markdown", "source": ["**Save Ouput**"], "metadata": {"_cell_guid": "62361098-b36c-4048-abe6-8c50ba7c1189", "_uuid": "981b5b6d5b6bcaf156c3b81edbb70512bf45e0c3"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": ["# ImageId,Label\n", "# pd.DataFrame({\"ImageId\": list(range(1,len(predicted_y)+1)), \"Label\": predicted_y}).to_csv('output.csv', index=False, header=True)"], "metadata": {"collapsed": true, "_cell_guid": "31cf2474-88d6-4031-bdca-4b03c6c42847", "_uuid": "c0b2df130808767bffb5b2229f739b116e7ba0dd"}}, {"outputs": [], "execution_count": null, "cell_type": "code", "source": [], "metadata": {"collapsed": true, "_cell_guid": "bc6a0357-8ef5-4e69-b0ef-2e1c4dc052f0", "_uuid": "e5475218e1302e9c8dea941e92572102ccacfccf"}}]}