{"cells":[{"metadata":{"trusted":true,"_uuid":"aea5b921eec200541236e321e5b5744abf6520c1","_kg_hide-output":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport tensorflow as tf\ntf.logging.set_verbosity(tf.logging.ERROR)\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\nfrom tensorflow.keras.layers import Dense, Convolution2D, MaxPooling2D, Flatten, BatchNormalization, Dropout, LeakyReLU\nfrom tensorflow.keras.models import Sequential\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73f6945157f901d740d8c5fb4b43f46766fdf020"},"cell_type":"code","source":"tf.enable_eager_execution()\ntfe = tf.contrib.eager","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f7d20fd20846612ef299e796500c839c7c9a037"},"cell_type":"code","source":"import tensorflow.contrib.eager as tfe ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccc9e40c209c31a44b548415510dc284649cba36"},"cell_type":"code","source":"# def create_model():\n#     model = Sequential()\n#     model.add(Convolution2D(filters = 16, kernel_size = 3, padding = 'same', input_shape = [28, 28, 1], activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Convolution2D(filters = 32, kernel_size = 3, padding = 'same', activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Convolution2D(filters = 64, kernel_size = 3, padding = 'same', activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Convolution2D(filters = 128, kernel_size = 3, padding = 'same', activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Convolution2D(filters = 256, kernel_size = 3, padding = 'same', activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Convolution2D(filters = 512, kernel_size = 3, padding = 'same', activation = 'relu'))\n#     model.add(MaxPooling2D(pool_size = (2,2)))\n#     model.add(BatchNormalization())\n#     model.add(Flatten())\n#     model.add(Dense(units = 100, activation = 'relu'))\n#     model.add(Dropout(0.5))\n#     model.add(Dense(units = 10))\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18ad0ad3b9114fdc43cc2331241e3035ad15a8ac"},"cell_type":"code","source":"def create_model():\n    model = Sequential()\n    model.add(Convolution2D(filters = 16, kernel_size = 3, padding = 'same', input_shape = [28, 28, 1]))\n    model.add(LeakyReLU(alpha=0.2))\n    model.add(MaxPooling2D(pool_size = (2,2)))\n    model.add(BatchNormalization())\n    model.add(Convolution2D(filters = 32, kernel_size = 3, padding = 'same'))\n    model.add(LeakyReLU(alpha=0.2))\n    model.add(MaxPooling2D(pool_size = (2,2)))\n    model.add(BatchNormalization())\n    model.add(Convolution2D(filters = 64, kernel_size = 3, padding = 'same'))\n    model.add(LeakyReLU(alpha=0.2))\n    model.add(MaxPooling2D(pool_size = (2,2)))\n    model.add(BatchNormalization())\n    model.add(Convolution2D(filters = 128, kernel_size = 3, padding = 'same'))\n    model.add(LeakyReLU(alpha=0.2))\n    model.add(MaxPooling2D(pool_size = (2,2)))\n    model.add(BatchNormalization())\n    model.add(Flatten())\n    model.add(Dense(units = 100))\n    model.add(LeakyReLU(alpha=0.2))\n    model.add(Dropout(0.5))\n    model.add(Dense(units = 10))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"354c9e04bc1baae89598a959ac810f1231300030"},"cell_type":"code","source":"model = create_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d1f7e1793945147357d62cb31c1e4bf243e9c2a"},"cell_type":"code","source":"model(np.zeros((10, 28, 28, 1), np.float32))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86378e42de2c5dbbfd9a441b573d4257f69f977e"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6340891f722b40b03506db282a261ca6e3e1d6df"},"cell_type":"code","source":"x_train = np.array(train.iloc[:,1:])\ny_train = np.array(train.iloc[:,0])\n\nx_test = np.array(test.iloc[:,:])\n# y_test = np.array(test.iloc[:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c095a7aff05c8ad474391b64bc800fbd637de8d"},"cell_type":"code","source":"y_train = tf.one_hot(y_train, 10)\n# y_test = tf.one_hot(y_test, 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f105c98efad9f9f23fe19c72a6b9ef4019ec93a"},"cell_type":"code","source":"batch_size = 256\ndataset = tf.data.Dataset.from_tensor_slices(\n        (x_train, y_train)).batch(batch_size)\ndataset_iter = tfe.Iterator(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c05b0710e4d87bc6adce93578f2cf16ff688506e"},"cell_type":"code","source":"#evaluate the loss\ndef loss(model, x, y):\n    prediction = model(x)\n    return tf.losses.softmax_cross_entropy(y, logits=prediction)\n\n#record the gradient with respect to the model variables \ndef grad(model, x, y):\n    with tf.GradientTape() as tape:\n        loss_value = loss(model, x, y)\n    return tape.gradient(loss_value, model.variables)\n\n#calcuate the accuracy of the model \ndef accuracy(model, x, y):\n\n    #prediction\n    yhat = model(x)\n\n    #get the labels of the predicted values \n    yhat = tf.argmax(yhat, 1).numpy()\n\n    #get the labels of the true values\n    y    = tf.argmax(y   , 1).numpy()\n    return np.sum(y == yhat)/len(y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d52884e2753275504c798ac99d31d6d865cbafaa"},"cell_type":"code","source":"#use Adam optimizer \noptimizer = tf.train.AdamOptimizer()\n\n#record epoch loss and accuracy  \nloss_history = tfe.metrics.Mean(\"loss\")\naccuracy_history = tfe.metrics.Mean(\"accuracy\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12da65db634eb6ff8a8d87e872bce722260d06d4"},"cell_type":"code","source":"def interval_mapping(image, from_min, from_max, to_min, to_max):\n    # map values from [from_min, from_max] to [to_min, to_max]\n    # image: input array\n    from_range = from_max - from_min\n    to_range = to_max - to_min\n    scaled = np.array((image - from_min) / float(from_range), dtype=np.float32)\n    return np.array(to_min + (scaled * to_range))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca905d9d79c3ecd42457cef2d115a549b68bf1da"},"cell_type":"code","source":"epoch = 0\nepochs = 3000\nwhile epoch < epochs:\n  #get next batch\n    try:\n        d = dataset_iter.next()\n    except StopIteration:\n        dataset_iter = tfe.Iterator(dataset)\n        d = dataset_iter.next()\n    \n    # Images\n    x_batch = d[0]\n    x_batch = tf.reshape(x_batch, shape=[-1, 28, 28, 1])\n    x_batch = interval_mapping(x_batch, 0, 1, -1, 1)\n    x = tf.convert_to_tensor(x_batch)\n    # Labels\n    y = d[1]\n    \n    \n\n    # Calculate derivatives of the input function with respect to its parameters.\n    grads = grad(model, x, y)\n\n    # Apply the gradient to the model\n    optimizer.apply_gradients(zip(grads, model.variables),\n                            global_step=tf.train.get_or_create_global_step())\n  \n    #record the current loss and accuracy   \n    loss_history(loss(model, x, y))\n    accuracy_history(accuracy(model, x, y))\n  \n    if epoch % 100 == 0:\n        print(\"epoch: {:d} Loss: {:.3f}, Acc: {:.3f}\".format(epoch, loss_history.result(), accuracy_history.result()))\n\n    #clear the history \n    loss_history.init_variables()\n    accuracy_history.init_variables()\n\n    epoch += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7369836c6adb4db08fe905121b50e7bc6fc1c1bb"},"cell_type":"code","source":"x_test = np.array(tf.reshape(x_test, shape=[-1, 28, 28, 1]))\nx_test_im = np.array(tf.reshape(x_test, shape=[-1, 28, 28]))\nx_test = interval_mapping(x_test, 0, 1, -1, 1)\nx_test = tf.convert_to_tensor(x_test)\npred_test = tf.argmax(model(x_test).numpy(), axis=1)\nprint(pred_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e499d9241771880f07c55a8a3fab32dbc440680c"},"cell_type":"code","source":"submissions=pd.DataFrame({\"ImageId\": list(range(1,len(pred_test.numpy())+1)),\n                         \"Label\": pred_test.numpy()})\nsubmissions.to_csv(\"submission_3.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"426d974651a92d575bf1a21981ef032fca5aa5dc"},"cell_type":"code","source":"op = pd.read_csv(\"submission_3.csv\")\nprint(op)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c55bc2e6160a2265b29729b6b635c2ad300bb17","_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}