{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-15T15:34:27.171969Z","iopub.execute_input":"2022-06-15T15:34:27.172401Z","iopub.status.idle":"2022-06-15T15:34:27.17748Z","shell.execute_reply.started":"2022-06-15T15:34:27.172364Z","shell.execute_reply":"2022-06-15T15:34:27.176602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:34:29.498537Z","iopub.execute_input":"2022-06-15T15:34:29.498957Z","iopub.status.idle":"2022-06-15T15:34:32.803865Z","shell.execute_reply.started":"2022-06-15T15:34:29.498925Z","shell.execute_reply":"2022-06-15T15:34:32.802846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:34:34.007928Z","iopub.execute_input":"2022-06-15T15:34:34.008324Z","iopub.status.idle":"2022-06-15T15:34:34.02977Z","shell.execute_reply.started":"2022-06-15T15:34:34.008292Z","shell.execute_reply":"2022-06-15T15:34:34.028893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = np.array(data)\nnp.random.shuffle(data)\nval_rate = 0.2\nval_num = int(data.shape[0] * val_rate)\n\nm, n=data.shape\n\nx_val = data[:val_num, 1:]\nt_val = data[:val_num, 0]\nx_train = data[val_num: , 1:]\nt_train = data[val_num: , 0]","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:34:36.154901Z","iopub.execute_input":"2022-06-15T15:34:36.155468Z","iopub.status.idle":"2022-06-15T15:34:36.989373Z","shell.execute_reply.started":"2022-06-15T15:34:36.155432Z","shell.execute_reply":"2022-06-15T15:34:36.988102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_params():\n    W1 = np.random.randn(784, 10) *0.01\n    b1 = np.zeros((1,10))\n    W2 = np.random.randn(10, 10) *0.01\n    b2 = np.zeros((1,10))\n    \n    return W1, b1, W2, b2\n\n\ndef relu(x):\n    return np.maximum(0, x)\n\n\n#def softmax(x):\n    #b = np.max(x, axis = 1)\n    #c = x[np.arange(x.shape[0])] - b[np.arange(len(b))]\n    #exp_x = np.exp(c)\n    #exp_sum = np.sum(exp_x, axis = 1)\n    #y = exp_x / (exp_sum)\n    #print(y[:3,:])\n    \n    #return y\n    \ndef softmax(x):\n    if x.ndim == 2:\n        x = x.T\n        x = x - np.max(x, axis=0)\n        y = np.exp(x) / np.sum(np.exp(x), axis=0)\n        return y.T \n\n    x = x - np.max(x) \n    return np.exp(x) / np.sum(np.exp(x))\n\n#def d_relu(x):\n    #return 1. * (x > 0)\n\n\nclass propagation:\n    def __init__(self):\n        self.A1 = None\n        self.Z1 = None\n        self.A2 = None\n        self.Y = None\n        self.T = None\n        self.x = None\n        self.W2 = None\n    \n    def forward(self, x, W1, b1, W2, b2):\n        self.x = x\n        self.W2 = W2\n        self.A1 = np.dot(self.x, W1) + b1\n        self.Z1 = relu(self.A1)\n        self.A2 = np.dot(self.Z1, W2) + b2\n        self.Y = softmax(self.A2)\n    \n        return self.Y\n\n\n    def backward(self, t):\n        self.T = one_hot(t)\n        m = self.T.size\n        dA2 = (self.Y - self.T) / m       #Nx10\n        dW2 = np.dot(self.Z1.T, dA2)   #10x10\n        db2 = np.sum(dA2, axis = 0)    \n        dZ1 = np.dot(dA2, self.W2.T)  #ㅒNx10\n        dA1 = dZ1 * np.array(self.Z1 > 0, dtype=int) #Nx10\n        #dA1 = dZ1 * d_relu(self.Z1) \n        db1 = np.sum(dA1, axis = 0) \n        dW1 = np.dot(self.x.T, dA1)    #784x10\n        \n        #print(np.shape(dA2))\n        #print(np.shape(dW2))\n        #print(np.shape(db2))   \n        #print(np.shape(dZ1))\n        #print(np.shape(dA1))\n        #print(np.shape(db1)) \n        #print(np.shape(dW1))\n        \n        return dW1, dW2, db1, db2\n        \n \n    \ndef one_hot(x):\n    x.reshape(1, x.size)\n    batch_size = len(x)\n    t = np.zeros((batch_size, 10))\n    t[np.arange(batch_size), x] = 1\n    \n    return t\n\ndef SGD(W1, b1, W2, b2, dW1, dW2, db1, db2, learning_rate):\n    db1.reshape(1, db1.size)\n    db2.reshape(1, db2.size)\n    lr = learning_rate\n    W1 = W1 - lr * dW1\n    W2 = W2 - lr * dW2\n    b1 = b1 - lr * db1\n    b2 = b2 - lr * db2\n    \n    return W1, b1, W2, b2\n\n\ndef prediction(Y):\n    return np.argmax(Y, axis = 1)\n\ndef accuracy(Y, t):\n    K = prediction(Y)\n    t.reshape(1, t.size)\n    return np.sum(K == t) / K.size\n\n\ndef train_network(x, t, iter, learning_rate):\n    W1, b1, W2, b2 = init_params()\n    prop = propagation()\n    \n    for i in range(iter):\n        Y = prop.forward(x, W1, b1, W2, b2)\n        dW1, dW2, db1, db2 = prop.backward(t)\n        W1, b1, W2, b2 = SGD(W1, b1, W2, b2, dW1, dW2, db1, db2, learning_rate)\n        if (i%10 == 0):\n            print('iteration: ', i)\n            print('accuracy: ', accuracy(Y, t))\n            \n            \n    return W1, b1, W2, b2\n\ndef test_network(x, t, W1, b1, W2, b2):\n    prop = propagation()\n    Y = prop.forward(x, W1, b1, W2, b2)\n    print(Y)\n    print(accuracy(Y, t))\n    return prediction(Y)\n\n\n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:34:39.117087Z","iopub.execute_input":"2022-06-15T15:34:39.117564Z","iopub.status.idle":"2022-06-15T15:34:39.144708Z","shell.execute_reply.started":"2022-06-15T15:34:39.117529Z","shell.execute_reply":"2022-06-15T15:34:39.143689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1, b1, W2, b2 = train_network(x_train, t_train, 500, 0.02)\n","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:34:44.844321Z","iopub.execute_input":"2022-06-15T15:34:44.844798Z","iopub.status.idle":"2022-06-15T15:39:00.506432Z","shell.execute_reply.started":"2022-06-15T15:34:44.844764Z","shell.execute_reply":"2022-06-15T15:39:00.504966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_network(x_val, t_val, W1, b1, W2, b2)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-06-15T15:39:03.473068Z","iopub.execute_input":"2022-06-15T15:39:03.47396Z","iopub.status.idle":"2022-06-15T15:39:03.548798Z","shell.execute_reply.started":"2022-06-15T15:39:03.473923Z","shell.execute_reply":"2022-06-15T15:39:03.54754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def submission(x, W1, b1, W2, b2):\n    prop = propagation()\n    Y = prop.forward(x, W1, b1, W2, b2)\n    \n    return prediction(Y)\n\ntest_data = pd.read_csv('../input/digit-recognizer/test.csv')\ntest_data = np.array(test_data)\nm, n = test_data.shape\n\nsubmission_dict = {\"ImageId\":np.array(range(1,m+1)),\"Label\":submission(test_data, W1, b1, W2, b2)}\nSubmission = pd.DataFrame(submission_dict)\nSubmission.head()\nSubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-06-15T16:20:58.573084Z","iopub.execute_input":"2022-06-15T16:20:58.573531Z","iopub.status.idle":"2022-06-15T16:21:00.945270Z","shell.execute_reply.started":"2022-06-15T16:20:58.573496Z","shell.execute_reply":"2022-06-15T16:21:00.944151Z"},"trusted":true},"execution_count":null,"outputs":[]}]}