{"cells":[{"metadata":{},"cell_type":"markdown","source":"# MNIST Scipy Convolutional Neural Network, Max Pool from Scratch","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Import libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np,sys\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom scipy.ndimage.filters import maximum_filter\nimport skimage.measure\nfrom scipy.signal import convolve2d","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preparation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/digit-recognizer/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = df.values[:,1:]\nlabels = df.values[:,0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.33, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test = X_train.reshape(X_train.shape[0], 28, 28), X_test.reshape(X_test.shape[0], 28, 28)\nX_train, X_test = X_train / 255.0, X_test / 255.0\ny_train, y_test = np.eye(10)[y_train], np.eye(10)[y_test]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Activation Functions & derivatives","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def relu(x):\n    mask = (x>0) * 1.0 \n    return x * mask\n\ndef drelu(x):\n    mask = (x>0) * 1.0 \n    return  mask","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sigmoid(z):\n    return 1.0 / (1 + np.exp(-z))\n\ndef dsigmoid(z):\n    return sigmoid(z) * (1-sigmoid(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def softmax(s): \n    exps = np.exp(s - np.max(s, axis=1, keepdims=True))\n    return exps/np.sum(exps, axis=1, keepdims=True)\n\ndef cross_entropy(pred, real):\n    n_samples = real.shape[0]\n    res = pred - real\n    return res/n_samples","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Error Function","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def error(pred, real):\n    n_samples = real.shape[0]\n    logp = - np.log(pred[np.arange(n_samples), real.argmax(axis=1)])\n    loss = np.sum(logp)/n_samples\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(12345)\n\nw1 = np.random.randn(169, 64)   * np.sqrt(1. /100)\nb1 = np.zeros((1, 64))          * np.sqrt(1. / 64)\nw2 = np.random.randn(64, 64)    * np.sqrt(1. / 64)\nb2 = np.zeros((1, 64))          * np.sqrt(1. / 128)\nw3 = np.random.randn(64, 10)    * np.sqrt(1. / 64)\nb3 = np.zeros((1, 10))          * np.sqrt(1. / 10)\n\nk = np.random.randn(3,3) * np.sqrt(1. / 14)\n\ntheta = k, w1, w2, w3, b1, b2, b3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Forward propagation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def forward(x, theta):\n    k, w1, w2, w3, b1, b2, b3 = theta\n\n    l1 = np.array([convolve2d(x[i], k, mode='valid') for i in range(len(x))])\n    l1a = relu(l1)\n    l1m = np.array([skimage.measure.block_reduce(l1a[i], (2,2), np.max) for i in range(len(l1a))])\n\n    z1 = l1m.reshape(l1m.shape[0], 169).dot(w1) + b1\n    a1 = sigmoid(z1)\n\n    z2 = a1.dot(w2) + b2\n    a2 = sigmoid(z2)\n\n    z3 = a2.dot(w3) + b3\n    a3 = softmax(z3)\n\n    return l1, l1a, l1m, z1, z2, z3, a1, a2, a3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Backward propagation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def backward(x, y, theta):\n    l1, l1a, l1m, z1, z2, z3, a1, a2, a3 = forward(x, theta)\n\n    da3 = cross_entropy(a3, y)\n    dz2 = np.dot(da3, w3.T)\n    da2 = dz2 * dsigmoid(a2)\n    dz1 = np.dot(da2, w2.T)\n    da1 = dz1 * dsigmoid(a1)\n    dz = np.dot(da1, w1.T)\n    dl1a = drelu(l1)\n\n    dw3 = np.dot(a2.T, da3)\n    db3 = np.sum(da3, axis=0, keepdims=True)\n\n    dw2 = np.dot(a1.T, da2)\n    db2 = np.sum(da2, axis=0)\n\n    dw1 = np.dot(l1m.reshape(169, x.shape[0]), da1)\n    db1 = np.sum(da1, axis=0)\n\n    masks = np.array([np.equal(l1a[i], l1m[i].repeat(2, axis=0).repeat(2, axis=1)).astype(int) for i in range(len(l1a))])\n    windows = np.array([masks[i] * dz[i].reshape(13,13).repeat(2, axis=0).repeat(2, axis=1) for i in range(len(masks))])\n\n    dks = np.array([np.rot90(convolve2d(x[i],np.rot90(windows[i] * dl1a[i],2 ),mode='valid'),2) for i in range(len(windows))])\n    dk = np.mean(dks, axis=0)\n\n    return dk, dw1, dw2, dw3, db1, db2, db3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def optimize(grads, theta, lr=0.3):\n    theta = tuple([theta[i] - (grads[i] * lr) for i in range(len(theta))])\n    \n    return theta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Only select the first 100 images just to test if the model trains well\nfeatures = np.array(np.array_split(X_train[:100], 5))\nlabels = np.array(np.array_split(y_train[:100], 5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"losses = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" for e in range(3001):\n    for i,x in enumerate(features):\n        x = x.reshape(x.shape[0], 28, 28)\n        grads = backward(x, labels[i], theta)\n        theta = optimize(grads, theta)\n\n    if e % 100 == 0:\n        preds = forward(X_train[:100].reshape(X_train[:100].shape[0], 28, 28), theta)[-1]\n        loss = error(preds, y_train[:100])\n        losses.append(loss)\n        print('Epoch: {0} -- Loss:  {1}'.format(e, loss))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Analyze loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(losses)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(img):\n    return np.argmax(forward(img.reshape(1, 28,28), theta)[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"image_to_predict = X_test[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# actual image of test set\nplt.imshow(image_to_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict image from the test set\npredict(image_to_predict)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}