{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:33:55.664765Z","iopub.execute_input":"2023-09-26T09:33:55.665644Z","iopub.status.idle":"2023-09-26T09:33:55.677812Z","shell.execute_reply.started":"2023-09-26T09:33:55.665606Z","shell.execute_reply":"2023-09-26T09:33:55.676898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:34:38.095504Z","iopub.execute_input":"2023-09-26T09:34:38.095870Z","iopub.status.idle":"2023-09-26T09:34:43.680067Z","shell.execute_reply.started":"2023-09-26T09:34:38.095840Z","shell.execute_reply":"2023-09-26T09:34:43.678906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train_data['label'].values\nX_train = train_data.drop(columns=['label']).values/255\nX_test = test_data.values/255","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:34:51.575877Z","iopub.execute_input":"2023-09-26T09:34:51.576308Z","iopub.status.idle":"2023-09-26T09:34:51.895179Z","shell.execute_reply.started":"2023-09-26T09:34:51.576270Z","shell.execute_reply":"2023-09-26T09:34:51.893974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(2,5, figsize=(12,5))\naxes = axes.flatten()\nidx = np.random.randint(0,42000,size=10)\nfor i in range(10):\n    axes[i].imshow(X_train[idx[i],:].reshape(28,28), cmap='gray')\n    axes[i].axis('off') # hide the axes ticks\n    axes[i].set_title(str(int(y_train[idx[i]])), color= 'black', fontsize=25)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:35:01.120691Z","iopub.execute_input":"2023-09-26T09:35:01.121225Z","iopub.status.idle":"2023-09-26T09:35:01.822605Z","shell.execute_reply.started":"2023-09-26T09:35:01.121182Z","shell.execute_reply":"2023-09-26T09:35:01.821484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def relu(x):\n    x[x<0]=0\n    return x","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:35:15.075667Z","iopub.execute_input":"2023-09-26T09:35:15.076120Z","iopub.status.idle":"2023-09-26T09:35:15.081317Z","shell.execute_reply.started":"2023-09-26T09:35:15.076063Z","shell.execute_reply":"2023-09-26T09:35:15.080067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def h(X,W,b):\n    '''\n    Hypothesis function: simple FNN with 1 hidden layer\n    Layer 1: input\n    Layer 2: hidden layer, with a size implied by the arguments W[0], b\n    Layer 3: output layer, with a size implied by the arguments W[1]\n    '''\n    # layer 1 = input layer\n    a1 = X\n    # layer 1 (input layer) -> layer 2 (hidden layer)\n    z1 = np.matmul(X, W[0]) + b[0]\n    \n    # add one more layer if applicable\n    \n    # layer 2 activation\n    a2 = relu(z1)\n    # layer 2 (hidden layer) -> layer 3 (output layer)\n    z2 = np.matmul(a2, W[1])\n    s = np.exp(z2)\n    total = np.sum(s, axis=1).reshape(-1,1)\n    sigma = s/total\n    # the output is a probability for each sample\n    return sigma","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:35:36.410405Z","iopub.execute_input":"2023-09-26T09:35:36.410801Z","iopub.status.idle":"2023-09-26T09:35:36.417664Z","shell.execute_reply.started":"2023-09-26T09:35:36.410771Z","shell.execute_reply":"2023-09-26T09:35:36.416536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(X_in,weights):\n    '''\n    Un-used cell for demo\n    activation function for the last FC layer: softmax function \n    Output: K probabilities represent an estimate of P(y=k|X_in;weights) for k=1,...,K\n    the weights has shape (n, K)\n    n: the number of features X_in has\n    n = X_in.shape[1]\n    K: the number of classes\n    K = 10\n    '''\n    \n    s = np.exp(np.matmul(X_in,weights))\n    total = np.sum(s, axis=1).reshape(-1,1)\n    return s / total","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:35:46.934096Z","iopub.execute_input":"2023-09-26T09:35:46.935104Z","iopub.status.idle":"2023-09-26T09:35:46.947769Z","shell.execute_reply.started":"2023-09-26T09:35:46.935063Z","shell.execute_reply":"2023-09-26T09:35:46.945405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss(y_pred,y_true):\n    '''\n    Loss function: cross entropy with an L^2 regularization\n    y_true: ground truth, of shape (N, )\n    y_pred: prediction made by the model, of shape (N, K) \n    N: number of samples in the batch\n    K: global variable, number of classes\n    '''\n    global K \n    K = 10\n    N = len(y_true)\n    # loss_sample stores the cross entropy for each sample in X\n    # convert y_true from labels to one-hot-vector encoding\n    y_true_one_hot_vec = (y_true[:,np.newaxis] == np.arange(K))\n    loss_sample = (np.log(y_pred) * y_true_one_hot_vec).sum(axis=1)\n    # loss_sample is a dimension (N,) array\n    # for the final loss, we need take the average\n    return -np.mean(loss_sample)","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:35:54.868536Z","iopub.execute_input":"2023-09-26T09:35:54.868889Z","iopub.status.idle":"2023-09-26T09:35:54.877768Z","shell.execute_reply.started":"2023-09-26T09:35:54.868861Z","shell.execute_reply":"2023-09-26T09:35:54.876523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def backprop(W,b,X,y,alpha=1e-4):\n    '''\n    Step 1: explicit forward pass h(X;W,b)\n    Step 2: backpropagation for dW and db\n    '''\n    K = 10\n    N = X.shape[0]\n    \n\n    a1 = X\n    z1 = np.matmul(X, W[0]) + b[0]\n    a2 = relu(z1)\n\n    z2 = np.matmul(a2, W[1])\n    s = np.exp(z2)\n    total = np.sum(s, axis=1).reshape(-1,1)\n    sigma = s/total\n\n    y_one_hot_vec = (y[:,np.newaxis] == np.arange(K))\n    delta2 = (sigma - y_one_hot_vec)\n    grad_W1 = np.matmul(a2.T, delta2)\n\n    delta1 = np.matmul(delta2, W[1].T)*(z1>0)\n    grad_W0 = np.matmul(X.T, delta1)\n    \n\n    \n    \n    dW = [grad_W0/N + alpha*W[0], grad_W1/N + alpha*W[1]]\n    db = [np.mean(delta1, axis=0)]\n    return dW, db","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:36:50.518957Z","iopub.execute_input":"2023-09-26T09:36:50.519306Z","iopub.status.idle":"2023-09-26T09:36:50.529244Z","shell.execute_reply.started":"2023-09-26T09:36:50.519275Z","shell.execute_reply":"2023-09-26T09:36:50.528095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eta = 5e-1\nalpha = 1e-6 # regularization\ngamma = 0.99 # RMSprop\neps = 1e-3 # RMSprop\nnum_iter = 1000 # number of iterations of gradient descent\nn_H = 256 # number of neurons in the hidden layer\nn = X_train.shape[1] # number of pixels in an image\nK = 10","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:37:01.873840Z","iopub.execute_input":"2023-09-26T09:37:01.874200Z","iopub.status.idle":"2023-09-26T09:37:01.879252Z","shell.execute_reply.started":"2023-09-26T09:37:01.874171Z","shell.execute_reply":"2023-09-26T09:37:01.878183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# initialization\nnp.random.seed(1127825)\nW = [1e-1*np.random.randn(n, n_H), 1e-1*np.random.randn(n_H, K)]\nb = [np.random.randn(n_H)]","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:37:09.684079Z","iopub.execute_input":"2023-09-26T09:37:09.684559Z","iopub.status.idle":"2023-09-26T09:37:09.706060Z","shell.execute_reply.started":"2023-09-26T09:37:09.684519Z","shell.execute_reply":"2023-09-26T09:37:09.705146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ngW0 = gW1 = gb0 = 1\n\nfor i in range(num_iter):\n    dW, db = backprop(W,b,X_train,y_train,alpha)\n    \n    gW0 = gamma*gW0 + (1-gamma)*np.sum(dW[0]**2)\n    etaW0 = eta/np.sqrt(gW0 + eps)\n    W[0] -= etaW0 * dW[0]\n    \n    gW1 = gamma*gW1 + (1-gamma)*np.sum(dW[1]**2)\n    etaW1 = eta/np.sqrt(gW1 + eps)\n    W[1] -= etaW1 * dW[1]\n    \n    gb0 = gamma*gb0 + (1-gamma)*np.sum(db[0]**2)\n    etab0 = eta/np.sqrt(gb0 + eps)\n    b[0] -= etab0 * db[0]\n    \n    if i % 500 == 0:\n        # sanity check 1\n        y_pred = h(X_train,W,b)\n        print(\"Cross-entropy loss after\", i+1, \"iterations is {:.8}\".format(\n              loss(y_pred,y_train)))\n        print(\"Training accuracy after\", i+1, \"iterations is {:.4%}\".format( \n              np.mean(np.argmax(y_pred, axis=1)== y_train)))\n        \n        # sanity check 2\n        print(\"gW0={:.4f} gW1={:.4f} gb0={:.4f}\\netaW0={:.4f} etaW1={:.4f} etab0={:.4f}\"\n              .format(gW0, gW1, gb0, etaW0, etaW1, etab0))\n        \n        # sanity check 3\n        print(\"|dW0|={:.5f} |dW1|={:.5f} |db0|={:.5f}\"\n             .format(np.linalg.norm(dW[0]), np.linalg.norm(dW[1]), np.linalg.norm(db[0])), \"\\n\")\n        \n        # reset RMSprop\n        gW0 = gW1 = gb0 = 1\n\ny_pred_final = h(X_train,W,b)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-26T09:40:15.521247Z","iopub.execute_input":"2023-09-26T09:40:15.521636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Final cross-entropy loss is {:.8}\".format(loss(y_pred_final,y_train)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Final training accuracy is {:.4%}\".format(np.mean(np.argmax(y_pred_final, axis=1)== y_train)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions\ny_pred_test = np.argmax(h(X_test,W,b), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': range(1,len(X_test)+1) ,'Label': y_pred_test })\nsubmission.to_csv(\"submission.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/working/submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}