{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport cv2\nimport tensorflow as tf\nfrom sklearn.utils import shuffle\nfrom sklearn.model_selection import train_test_split\nimport os\nfrom sklearn.model_selection import StratifiedShuffleSplit\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"collapsed":true},"cell_type":"code","source":"#load test data\nwith open(\"../input/test.csv\",\"rb\") as f:\n temp_data = pd.read_csv(f)\n#test_labels = np.asarray(temp_data[\"label\"])\ntest_data = np.asarray(temp_data)\n\n#load train data and labels\nwith open(\"../input/train.csv\",\"rb\") as f:\n temp_data = pd.read_csv(f)\ntrain_labels = np.asarray(temp_data[\"label\"])\ntrain_data = np.asarray(temp_data.drop(columns=\"label\"))\n\n#val_count = np.floor_divide(30*len(train_labels),100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18e0ab6683ff8676e8489680ae99787604c9cb37","collapsed":true},"cell_type":"code","source":"split = StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)\nfor train_index,test_index in split.split(train_data,train_labels):\n strat_train_data, strat_val_data = train_data[train_index], train_data[test_index]\n strat_train_labels, strat_val_labels = train_labels[train_index], train_labels[test_index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1ee31c156904453bc9dc864c3eaeed66c873ec56"},"cell_type":"code","source":"#define image dimensions which is 28x28x1 for MNIST dataset\nheight = 28\nwidth = 28\nchannel = 1\nproj_dir = \"../working/character_classification_akj/\"\ncheckpoint_path = proj_dir + \"intermediate_checkpoint.ckpt\"\ncheckpoint_epoch_path = checkpoint_path+\".epoch\"\nfinal_model_path = proj_dir + \"my_model\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1fe80bdf63b7b8316431f5795ed01d5d1f1f5459"},"cell_type":"code","source":"def fetch_batch(feature_set, labels, batch_size):\n p = np.random.permutation(len(feature_set))\n return (feature_set[p][:batch_size,:], labels[p][:batch_size])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aac3a9639a31f92b5ea4776cc6b478ffd7bd8fe3","collapsed":true},"cell_type":"code","source":"tf.reset_default_graph()\nX = tf.placeholder(tf.float32, [None,height*width*channel], name=\"X\")\ny = tf.placeholder(tf.int32, [None], name=\"y\")\ntraining = tf.placeholder_with_default(False,shape=(), name = \"training_variable\")\nlearning_rate = tf.placeholder_with_default(0.01,shape=(), name= \"learning_rate\")\nglobal_step_tensor = tf.Variable(0,trainable=False, name= \"global_step\")\n\n\ninput_layer = tf.reshape(X, [-1,height,width,channel], name=\"Input_layer\")\n\nconv1 = tf.layers.conv2d(inputs= input_layer,\n filters= 32,\n kernel_size= [5,5],\n padding= \"same\",\n activation= tf.nn.relu,\n name= \"conv_1\")\n\npool1 = tf.layers.max_pooling2d(inputs= conv1,\n pool_size= [2,2],\n strides= 2,\n name= \"pool_1\")\n\nconv2 = tf.layers.conv2d(inputs= pool1,\n filters= 64,\n kernel_size= [5,5],\n padding= \"same\",\n activation= tf.nn.relu,\n name= \"conv_2\")\n \npool2 = tf.layers.max_pooling2d(inputs= conv2,\n pool_size= [2,2],\n strides= 2,\n name= \"pool_2\")\n \npool2_flat = tf.reshape(pool2, [-1,7*7*64])\n \ndense = tf.layers.dense(inputs= pool2_flat,\n units= 1024,\n activation= tf.nn.relu)\n\ndropout = tf.layers.dropout(dense,\n rate=0.3,\n training=training,\n name=\"Dropout\")\n\nlogits = tf.layers.dense(inputs= dropout, units= 10)\n#classes = tf.argmax(logits,axis=1)\nxentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,\n labels=y,\n name=\"xentropy\")\nloss = tf.reduce_mean(xentropy,\n name=\"loss\")\n\noptimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\ntraining_op = optimizer.minimize(loss,global_step=global_step_tensor)\n\ncorrect = tf.nn.in_top_k(logits, y,1)\naccuracy = tf.reduce_mean(tf.cast(correct, dtype=tf.float32))\ninit = tf.global_variables_initializer()\nsaver = tf.train.Saver()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76f6933bf4a520a106f2f57a0da7c2bfea2b183d","collapsed":true},"cell_type":"code","source":"learning_rate_array = 0.0001*np.exp(-20000*np.linspace(0.000001,0.001,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fefee1437641138acd17abdf86590318dbd7613"},"cell_type":"code","source":"train_loss_list = []\ntrain_accuracy_list = []\nval_loss_list = []\nval_accuracy_list = []\nold_loss = np.inf\nnew_loss = 0\nearly_stop_count = 0\n\nn_epoch = 10\nn_iter = 1000\nbatch_size = 200\nwith tf.Session() as sess:\n if os.path.isfile(checkpoint_epoch_path):\n with open(checkpoint_epoch_path) as f:\n start_epoch = int(f.read())\n print(\"Training was interupted resuming from epoch \", start_epoch)\n saver.restore(sess,checkpoint_path)\n else:\n start_epoch = 0\n init.run()\n os.mkdir(\"../working/character_classification_akj\")\n for epoch in range(start_epoch,n_epoch):\n learning_rate_epoch = learning_rate_array[epoch]\n for iteration in range(n_iter):\n X_batch, y_batch = fetch_batch(strat_train_data, strat_train_labels, batch_size)\n sess.run(training_op, feed_dict={X:X_batch,\n y:y_batch,\n training:True,\n learning_rate:learning_rate_epoch})\n new_loss = sess.run(loss,feed_dict={X:strat_val_data,y:strat_val_labels})\n if new_loss500:\n print(\"Early stopping satisfied with best epoch \",best_epoch,\n \" and best iteration \",iteration)\n current_train_loss, current_train_accu = sess.run([loss,accuracy],\n feed_dict={X:strat_train_data,\n y:strat_train_labels})\n current_val_loss, current_val_accu = sess.run([loss,accuracy],\n feed_dict={X:strat_val_data,\n y:strat_val_labels})\n print(\"Current train loss: \",current_train_loss,\n \" Current train accuracy: \", current_train_accu)\n print()\n print(\"Current val loss: \",current_val_loss,\n \" Current val accuracy: \", current_val_accu)\n print()\n print(\"Best model loss: \", old_loss)\n break\n if early_stop_count>50:\n break\n if epoch%1==0:\n print(\"Saving checkpoint for epoch \",epoch)\n saver.save(sess,checkpoint_path)\n with open(checkpoint_epoch_path,\"wb\") as f:\n f.write(b'%d' % (epoch+1))\n train_loss, train_accuracy = sess.run([loss, accuracy],\n feed_dict={X:X_batch, y:y_batch})\n train_loss_list.append(train_loss)\n train_accuracy_list.append(train_accuracy)\n print(\"Epoch: \",epoch) \n print(\"Training loss: \",train_loss,\" Training accuracy: \",train_accuracy)\n print()\n #saver.save(sess,final_model_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f35048a4025d4ed5db3c47948c7710682b89561"},"cell_type":"code","source":"with tf.Session() as sess:\n saver.restore(sess,final_model_path)\n predict = sess.run(logits,feed_dict={X:test_data})\n predict_class = sess.run(tf.argmax(predict,axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"78d718406a0864e6911b9ea32d84d352a322ba54"},"cell_type":"code","source":"output_file = \"submission.csv\"\nwith open(output_file, 'w') as f :\n f.write('ImageId,Label\\n')\n for i in range(len(predict_class)) :\n f.write(\"\".join([str(i+1),',',str(predict_class[i]),'\\n']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d94d98bbbf0358a9cfca9cff6814930ed4d3be19"},"cell_type":"code","source":"os.listdir(\"./\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a852aa4b3bd1da2f7899ae23a8a91c5f2bd2f2bd","collapsed":true},"cell_type":"markdown","source":"delete_path = \"../working/character_classification_akj/\"\nfor i in os.listdir(delete_path):\n if os.path.isfile(delete_path+i):\n os.remove(delete_path+i)\n else: \n os.rmdir(delete_path+i) \nos.rmdir(\"../working/character_classification_akj/\")"},{"metadata":{"trusted":true,"_uuid":"ff1eef80675cca311c6c0d64862c8b9f860a6e1e"},"cell_type":"code","source":"plt.plot(0.01*np.exp(-20000*np.linspace(0.000001,0.0001,10)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b51b6aa8172655887e1baf24ff37fc8d8ab9ca5"},"cell_type":"code","source":"0.0001*np.exp(-20000*np.linspace(0.000001,0.001,10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"4d74e15ec3b4abf86e35858b33af2bd8fec96bb9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}