{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cv2\nimport numpy as np \nimport pandas as pd\nfrom matplotlib import pyplot as plt\nfrom scipy.special import softmax","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:07.834209Z","iopub.execute_input":"2022-02-04T14:14:07.834509Z","iopub.status.idle":"2022-02-04T14:14:07.839794Z","shell.execute_reply.started":"2022-02-04T14:14:07.834476Z","shell.execute_reply":"2022-02-04T14:14:07.838762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:07.841526Z","iopub.execute_input":"2022-02-04T14:14:07.841891Z","iopub.status.idle":"2022-02-04T14:14:10.585565Z","shell.execute_reply.started":"2022-02-04T14:14:07.841860Z","shell.execute_reply":"2022-02-04T14:14:10.584238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np_data = np.array(pd_data) # Convert data to np array","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:10.586938Z","iopub.execute_input":"2022-02-04T14:14:10.587219Z","iopub.status.idle":"2022-02-04T14:14:10.698204Z","shell.execute_reply.started":"2022-02-04T14:14:10.587183Z","shell.execute_reply":"2022-02-04T14:14:10.697260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(np_data)\nm, n = np_data.shape\n#print(m,n)\nnp.random.shuffle(np_data)\n\ndata_inference = np_data[0:1000].T # Transpose for change data, each column is image\ninference_labels = data_inference[0]\ninference_data = data_inference[1:n]\ninference_data = inference_data / 255.\n\ndata_train = np_data[1000:m].T # 1000 to last image is train dataset\ntrain_labels = data_train[0]\ntrain_data = data_train[1:n]\ntrain_data = train_data / 255.","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:10.699478Z","iopub.execute_input":"2022-02-04T14:14:10.699763Z","iopub.status.idle":"2022-02-04T14:14:11.399350Z","shell.execute_reply.started":"2022-02-04T14:14:10.699730Z","shell.execute_reply":"2022-02-04T14:14:11.398727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_network():\n    W1 = np.random.rand(10, 784) - 0.5\n    b1 = np.random.rand(10,1) - 0.5\n    W2 = np.random.rand(10,10) - 0.5\n    b2 = np.random.rand(10,1) - 0.5\n    return W1,b1,W2,b2\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.401606Z","iopub.execute_input":"2022-02-04T14:14:11.401886Z","iopub.status.idle":"2022-02-04T14:14:11.407243Z","shell.execute_reply.started":"2022-02-04T14:14:11.401854Z","shell.execute_reply":"2022-02-04T14:14:11.406326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#W1,b1,W2,b2 = create_network()\n#print(W1,b1,W2,b2)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.408381Z","iopub.execute_input":"2022-02-04T14:14:11.408684Z","iopub.status.idle":"2022-02-04T14:14:11.419915Z","shell.execute_reply.started":"2022-02-04T14:14:11.408629Z","shell.execute_reply":"2022-02-04T14:14:11.419124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def forward_propagation(W1,b1,W2,b2,train_data): # Returns Z1,A1,Z2,A2\n    Z1 = W1.dot(train_data) + b1 # Z1 = W1.A0 + b1\n    A1 = np.maximum(Z1,0) # This is ReLU function A1 = ReLU(Z1)\n    Z2 = W2.dot(A1) + b2 # Z2 = W2.A1 + b2\n    A2 = np.exp(Z2) / sum(np.exp(Z2)) # A2 softmax(Z2) softmax function that transform value 0 to 1\n    return Z1,A1,Z2,A2\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.421305Z","iopub.execute_input":"2022-02-04T14:14:11.421813Z","iopub.status.idle":"2022-02-04T14:14:11.431657Z","shell.execute_reply.started":"2022-02-04T14:14:11.421761Z","shell.execute_reply":"2022-02-04T14:14:11.430937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Z1,A1,Z2,A2 = forward_propagation(W1,b1,W2,b2, train_data)\n#print(A2)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.433134Z","iopub.execute_input":"2022-02-04T14:14:11.433607Z","iopub.status.idle":"2022-02-04T14:14:11.444467Z","shell.execute_reply.started":"2022-02-04T14:14:11.433574Z","shell.execute_reply":"2022-02-04T14:14:11.443718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def relu_derivative(Z):\n    return (Z > 0).astype(int)\n\ndef hot_encode_labels(Y): # #To organize labels ie: label 0 represents [1,0,0,...,0]\n    one_hot_Y = np.zeros((Y.size, Y.max() + 1))\n    one_hot_Y[np.arange(Y.size), Y] = 1\n    one_hot_Y = one_hot_Y.T\n    return one_hot_Y\n\ndef backward_propagation(Z1, A1, Z2, A2, W1, W2, X, Y):\n    encoded_labels = hot_encode_labels(Y)\n    dZ2 = A2 - encoded_labels # Error of second layer\n    dW2 = 1/m * dZ2.dot(A1.T)\n    db2 = 1/m * np.sum(dZ2)\n    dZ1 = W2.T.dot(dZ2) * relu_derivative(Z1)\n    dW1 = 1/m * dZ2.dot(X.T)\n    db1 = 1/m * np.sum(dZ1)\n    return dW1, db1, dW2, db2\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.446010Z","iopub.execute_input":"2022-02-04T14:14:11.446534Z","iopub.status.idle":"2022-02-04T14:14:11.457258Z","shell.execute_reply.started":"2022-02-04T14:14:11.446486Z","shell.execute_reply":"2022-02-04T14:14:11.456095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dZ2, dW2, db2, dZ1, dW1, db1 = backward_propagation(Z1, A1, Z2, A2, train_data, hot_encode_labels, W2, W1)\n#print(dZ2, dW2, db2, dZ1, dW1, db1)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.459151Z","iopub.execute_input":"2022-02-04T14:14:11.459457Z","iopub.status.idle":"2022-02-04T14:14:11.472362Z","shell.execute_reply.started":"2022-02-04T14:14:11.459423Z","shell.execute_reply":"2022-02-04T14:14:11.471139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_error_parameters(W1,b1,W2,b2,dW1,db1,dW2,db2, learning_rate):\n    W1 = W1 - learning_rate * dW1\n    b1 = b1 - learning_rate * db1\n    W2 = W2 - learning_rate * dW2\n    b2 = b2 - learning_rate * db2\n    return W1, b1, W2, b2\n    ","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.473760Z","iopub.execute_input":"2022-02-04T14:14:11.474599Z","iopub.status.idle":"2022-02-04T14:14:11.483924Z","shell.execute_reply.started":"2022-02-04T14:14:11.474561Z","shell.execute_reply":"2022-02-04T14:14:11.482887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#W1, b1, W2, b2 = update_error_parameters(W1,b1,W2,b2,dW1,db1,dW2,db2, 0.500)\n#print(W1, b1, W2, b2)\ndef get_predictions(A2):\n    return np.argmax(A2, 0)\n\ndef get_accuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y) / Y.size\n\ndef gradient_descent(X, Y, learning_rate, iterations):\n    W1, b1, W2, b2 = create_network()\n    for i in range(iterations):\n        Z1, A1, Z2, A2 = forward_propagation(W1, b1, W2, b2, X)\n        dW1, db1, dW2, db2 = backward_propagation(Z1, A1, Z2, A2, W1, W2, X, Y)\n        W1, b1, W2, b2 = update_error_parameters(W1, b1, W2, b2, dW1, db1, dW2, db2, learning_rate)\n        if i % 100 == 0:\n            print(\"Iteration: \", i)\n            predictions = get_predictions(A2)\n            print(get_accuracy(predictions, Y))\n    return W1, b1, W2, b2\n\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2022-02-04T14:14:11.485251Z","iopub.execute_input":"2022-02-04T14:14:11.485762Z","iopub.status.idle":"2022-02-04T14:14:11.497872Z","shell.execute_reply.started":"2022-02-04T14:14:11.485728Z","shell.execute_reply":"2022-02-04T14:14:11.496978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1, b1, W2, b2 = gradient_descent(train_data, train_labels, 0.2, 2000)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:14:11.499309Z","iopub.execute_input":"2022-02-04T14:14:11.499540Z","iopub.status.idle":"2022-02-04T14:19:05.600377Z","shell.execute_reply.started":"2022-02-04T14:14:11.499512Z","shell.execute_reply":"2022-02-04T14:19:05.599324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_predictions(X, W1, b1, W2, b2):\n    Z1, A1, Z2, A2 = forward_propagation(W1, b1, W2, b2, X)\n    predictions = get_predictions(A2)\n    return predictions\n\ndef test_prediction(index, W1, b1, W2, b2):\n    current_image = inference_data[:, index, None]\n    prediction = make_predictions(inference_data[:, index, None], W1, b1, W2, b2)\n    label = inference_labels[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n    \n    current_image = current_image.reshape((28, 28)) * 255\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()\n    \ntest_prediction(33, W1, b1, W2, b2)\ntest_prediction(234, W1, b1, W2, b2)\ntest_prediction(112, W1, b1, W2, b2)\ntest_prediction(131, W1, b1, W2, b2)\ntest_prediction(9, W1, b1, W2, b2)\ntest_prediction(99, W1, b1, W2, b2)\ntest_prediction(24, W1, b1, W2, b2)\ntest_prediction(67, W1, b1, W2, b2)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T14:19:05.608340Z","iopub.execute_input":"2022-02-04T14:19:05.612272Z","iopub.status.idle":"2022-02-04T14:19:07.161895Z","shell.execute_reply.started":"2022-02-04T14:19:05.609331Z","shell.execute_reply":"2022-02-04T14:19:07.160931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}