{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[{"output_type":"stream","text":"['test.csv', 'train.csv', 'sample_submission.csv']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# Read the data\ntrain_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv(\"../input/test.csv\")\n\n","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\n\n# Set up the data\ny_train = train_data['label'].values\nX_train = train_data.drop(columns=['label']).values/255\nX_test = test_data.values/255\n\n","execution_count":3,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# relu activation function\ndef relu(x):\n    return x*(x>0)\n\n","execution_count":4,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# hypothesis function: simple FNN with 1 hidden layer\ndef h(X,W,b):\n    # layer 1 = input layer\n    a1 = X\n    # layer 1 (input layer) -> layer 2 (hidden layer)\n    z1 = np.matmul(X, W[0]) + b[0]\n    # layer 2 activation\n    a2 = relu(z1)\n    # layer 2 (hidden layer) -> layer 3 (output layer)\n    z2 = np.matmul(a2, W[1])\n    s = np.exp(z2)\n    total = np.sum(s, axis=1).reshape(-1,1)\n    sigma = s/total\n    # the output is a probability for each sample\n    return sigma\n\n","execution_count":5,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Un-used cell for demo\n# activation function for the last FC layer: softmax function \n# Output: K probabilities represent an estimate of P(y=k|X_in;weights) for k=1,...,K\ndef softmax(X_in,weights):\n    # the weights has shape (n, K)\n    # n: the number of features X_in has\n    # n = X_in.shape[1]\n    # K: the number of classes\n    # K = 10\n    s = np.exp(np.matmul(X_in,weights))\n    total = np.sum(s, axis=1).reshape(-1,1)\n    return s / total","execution_count":6,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# loss function: cross entropy with L^2 regularization\ndef loss(W,b,X,y,alpha=1e-4):\n    # N: number of samples in the batch\n    K = 10\n    N = X.shape[0]\n    # KL_sample stores the cross entropy for each sample in X\n    KL_sample = np.zeros(N)\n    for k in range(K):\n        KL_sample += np.log(h(X,W,b))[:,k] * (y == k)\n    # KL_sample is a dimension (N,) array\n    reg = 0.5*alpha*(np.sum(W[1]**2) + np.sum(W[0]**2))\n    return -np.mean(KL_sample) + reg","execution_count":7,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef backprop(W,b,X,y,alpha=1e-4):\n    '''\n    Step 1: explicit forward pass h(X;W,b)\n    Step 2: backpropagation for dW and db\n    '''\n    K = 10\n    N = X.shape[0]\n    \n    ### Step 1:\n    # layer 1 = input layer\n    a1 = X\n    # layer 1 (input layer) -> layer 2 (hidden layer)\n    z1 = np.matmul(X, W[0]) + b[0]\n    # layer 2 activation\n    a2 = relu(z1)\n    # layer 2 (hidden layer) -> layer 3 (output layer)\n    z2 = np.matmul(a2, W[1])\n    s = np.exp(z2)\n    total = np.sum(s, axis=1).reshape(-1,1)\n    sigma = s/total\n    \n    ### Step 2:\n    # layer 2->layer 3 weights' derivative\n    # delta2 is \\partial L/partial z2\n    delta2 = np.zeros([N,K])\n    for k in range(K):\n        delta2[:,k] = (sigma[:,k] - (y==k))\n    grad_W1 = np.matmul(a2.T, delta2)\n    # layer 1->layer 2 weights' derivative\n    # delta1 is \\partial a2/partial z1\n    # layer 2 activation's (weak) derivative is 1*(z1>0)\n    delta1 = np.matmul(delta2, W[1].T)*(z1>0)\n    grad_W0 = np.matmul(X.T, delta1)\n    # no derivative for layer 1\n    \n    \n    dW = [grad_W0/N + alpha*W[0], grad_W1/N + alpha*W[1]]\n    db = [np.mean(delta1, axis=0)]\n    # dW[0] is W[0]'s derivative, and dW[1] is W[1]'s derivative; similar for db\n    return dW, db\n\n","execution_count":8,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyper-parameters and network\neta = 5e-1\nalpha = 1e-6 # regularization\ngamma = 0.99 # RMSprop\neps = 1e-3 # RMSprop\nnum_iter = 2500 # number of iterations of gradient descent\nn_H = 256 # number of neurons in the hidden layer\nn = X_train.shape[1] # number of pixels in an image\nK = 10","execution_count":9,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# initialization\nnp.random.seed(42)\nW = [1e-1*np.random.randn(n, n_H), 1e-1*np.random.randn(n_H, K)]\nb = [np.random.randn(n_H)]","execution_count":10,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gW0 = gW1 = gb0 = 1\n\nfor i in range(num_iter):\n    dW, db = backprop(W,b,X_train,y_train,alpha)\n    \n    gW0 = gamma*gW0 + (1-gamma)*np.sum(dW[0]**2)\n    etaW0 = eta/np.sqrt(gW0 + eps)\n    W[0] -= etaW0 * dW[0]\n    \n    gW1 = gamma*gW1 + (1-gamma)*np.sum(dW[1]**2)\n    etaW1 = eta/np.sqrt(gW1 + eps)\n    W[1] -= etaW1 * dW[1]\n    \n    gb0 = gamma*gb0 + (1-gamma)*np.sum(db[0]**2)\n    etab0 = eta/np.sqrt(gb0 + eps)\n    b[0] -= etab0 * db[0]\n    \n    if i % 500 == 0:\n        # sanity check 1\n        print(\"loss after\", i+1, \"iterations is {:.8}\".format(\n              loss(W,b,X_train,y_train,alpha)))\n        print(\"training accuracy after\", i+1, \"iterations is {:.4%}\".format( \n              np.mean(np.argmax(h(X_train,W,b), axis=1)== y_train)))\n        \n        # sanity check 2\n        print(\"gW0={:.4f} gW1={:.4f} gb0={:.4f}\\netaW0={:.4f} etaW1={:.4f} etab0={:.4f}\"\n              .format(gW0, gW1, gb0, etaW0, etaW1, etab0))\n        \n        # sanity check 3\n        print(\"|dW0|={:.5f} |dW1|={:.5f} |db0|={:.5f}\"\n             .format(np.linalg.norm(dW[0]), np.linalg.norm(dW[1]), np.linalg.norm(db[0])), \"\\n\")\n        \n        # reset RMSprop\n        gW0 = gW1 = gb0 = 1\n\nprint(\"final loss is {:.8}\".format(loss(W,b,X_train,y_train,alpha)))\nprint(\"final training accuracy is {:.4%}\".format(np.mean(np.argmax(h(X_train,W,b), axis=1)== y_train)))","execution_count":11,"outputs":[{"output_type":"stream","text":"loss after 1 iterations is 7.68388\ntraining accuracy after 1 iterations is 14.1238%\ngW0=1.0283 gW1=1.1513 gb0=0.9907\netaW0=0.4928 etaW1=0.4658 etab0=0.5021\n|dW0|=1.95603 |dW1|=4.01615 |db0|=0.26572 \n\nloss after 501 iterations is 0.19777055\ntraining accuracy after 501 iterations is 94.0643%\ngW0=0.0715 gW1=0.0345 gb0=0.0081\netaW0=1.8570 etaW1=2.6532 etab0=5.2523\n|dW0|=0.26741 |dW1|=0.14707 |db0|=0.04088 \n\nloss after 1001 iterations is 0.071777433\ntraining accuracy after 1001 iterations is 97.9143%\ngW0=0.1497 gW1=0.0328 gb0=0.0098\netaW0=1.2879 etaW1=2.7191 etab0=4.8173\n|dW0|=0.03186 |dW1|=0.01016 |db0|=0.00413 \n\nloss after 1501 iterations is 0.042340791\ntraining accuracy after 1501 iterations is 98.8667%\ngW0=0.4012 gW1=0.1146 gb0=0.0172\netaW0=0.7884 etaW1=1.4706 etab0=3.7069\n|dW0|=0.01293 |dW1|=0.00539 |db0|=0.00146 \n\nloss after 2001 iterations is 0.030471707\ntraining accuracy after 2001 iterations is 99.2905%\ngW0=0.2622 gW1=0.0615 gb0=0.0131\netaW0=0.9746 etaW1=1.9999 etab0=4.2079\n|dW0|=0.01058 |dW1|=0.00434 |db0|=0.00111 \n\nfinal loss is 0.029874464\nfinal training accuracy is 99.2857%\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# predictions\ny_pred = np.argmax(h(X_test,W,b), axis=1)\n\n","execution_count":12,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# Generating submission using pandas for grading\nsubmission = pd.DataFrame({'ImageId': range(1,len(X_test)+1) ,'Label': y_pred })\nsubmission.to_csv(\"simplemnist_result.csv\",index=False)\n\n","execution_count":13,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}