{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd ","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-04-13T18:01:07.715724Z","iopub.execute_input":"2023-04-13T18:01:07.716058Z","iopub.status.idle":"2023-04-13T18:01:07.721081Z","shell.execute_reply.started":"2023-04-13T18:01:07.716029Z","shell.execute_reply":"2023-04-13T18:01:07.719732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"https://docs.google.com/document/d/1m6R-kDJEXN-qQPlA4QhWtfQkuw8kl586K4TG2BoqmAQ/edit?usp=sharing","metadata":{}},{"cell_type":"code","source":"data=pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:07.723345Z","iopub.execute_input":"2023-04-13T18:01:07.724263Z","iopub.status.idle":"2023-04-13T18:01:09.471467Z","shell.execute_reply.started":"2023-04-13T18:01:07.724192Z","shell.execute_reply":"2023-04-13T18:01:09.469907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.473159Z","iopub.execute_input":"2023-04-13T18:01:09.473506Z","iopub.status.idle":"2023-04-13T18:01:09.494769Z","shell.execute_reply.started":"2023-04-13T18:01:09.473476Z","shell.execute_reply":"2023-04-13T18:01:09.493767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m,n=data.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.496093Z","iopub.execute_input":"2023-04-13T18:01:09.496380Z","iopub.status.idle":"2023-04-13T18:01:09.502560Z","shell.execute_reply.started":"2023-04-13T18:01:09.496354Z","shell.execute_reply":"2023-04-13T18:01:09.501463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=np.array(data)\nnp.random.shuffle(data)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.505469Z","iopub.execute_input":"2023-04-13T18:01:09.505793Z","iopub.status.idle":"2023-04-13T18:01:09.868156Z","shell.execute_reply.started":"2023-04-13T18:01:09.505765Z","shell.execute_reply":"2023-04-13T18:01:09.866906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*now we convert the data into array format since we will be implementing neural network architechture without tensorflow/keras*","metadata":{}},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.869673Z","iopub.execute_input":"2023-04-13T18:01:09.870012Z","iopub.status.idle":"2023-04-13T18:01:09.876579Z","shell.execute_reply.started":"2023-04-13T18:01:09.869977Z","shell.execute_reply":"2023-04-13T18:01:09.875713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.877693Z","iopub.execute_input":"2023-04-13T18:01:09.877995Z","iopub.status.idle":"2023-04-13T18:01:09.889347Z","shell.execute_reply.started":"2023-04-13T18:01:09.877964Z","shell.execute_reply":"2023-04-13T18:01:09.888572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"m , n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.890394Z","iopub.execute_input":"2023-04-13T18:01:09.890674Z","iopub.status.idle":"2023-04-13T18:01:09.900675Z","shell.execute_reply.started":"2023-04-13T18:01:09.890646Z","shell.execute_reply":"2023-04-13T18:01:09.899636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"*Shuffling data serves the purpose of reducing variance and making sure that models remain general and overfit less.*","metadata":{}},{"cell_type":"markdown","source":"We transpose the given data to","metadata":{}},{"cell_type":"code","source":"batch=data[0:1000].T #transposing the data set for matrix calculations\nbatch","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.902127Z","iopub.execute_input":"2023-04-13T18:01:09.902459Z","iopub.status.idle":"2023-04-13T18:01:09.914457Z","shell.execute_reply.started":"2023-04-13T18:01:09.902431Z","shell.execute_reply":"2023-04-13T18:01:09.913321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_batch=batch[0] #first row is the target variable","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.915687Z","iopub.execute_input":"2023-04-13T18:01:09.915937Z","iopub.status.idle":"2023-04-13T18:01:09.923579Z","shell.execute_reply.started":"2023-04-13T18:01:09.915913Z","shell.execute_reply":"2023-04-13T18:01:09.922299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_batch=batch[1:n]\nx_batch = x_batch / 255","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.924909Z","iopub.execute_input":"2023-04-13T18:01:09.925303Z","iopub.status.idle":"2023-04-13T18:01:09.933061Z","shell.execute_reply.started":"2023-04-13T18:01:09.925264Z","shell.execute_reply":"2023-04-13T18:01:09.932066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train=data[1000:m].T\ny_train=data_train[0]\nx_train=data_train[1:n]\nx_train = x_train / 255\nx_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:09.934620Z","iopub.execute_input":"2023-04-13T18:01:09.934976Z","iopub.status.idle":"2023-04-13T18:01:10.002335Z","shell.execute_reply.started":"2023-04-13T18:01:09.934938Z","shell.execute_reply":"2023-04-13T18:01:10.000852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_,m_train=x_train.shape #we have 784 pixel values (number of features (n) ) and 41000 training examples(m) with us","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.006450Z","iopub.execute_input":"2023-04-13T18:01:10.006809Z","iopub.status.idle":"2023-04-13T18:01:10.011081Z","shell.execute_reply.started":"2023-04-13T18:01:10.006778Z","shell.execute_reply":"2023-04-13T18:01:10.009938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train #we have 41000 target variables which matches with the number of training examples #number of elemnts in the row","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.015869Z","iopub.execute_input":"2023-04-13T18:01:10.016129Z","iopub.status.idle":"2023-04-13T18:01:10.023622Z","shell.execute_reply.started":"2023-04-13T18:01:10.016104Z","shell.execute_reply":"2023-04-13T18:01:10.022727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def init_params():\n    W1 = np.random.rand(10, 784) - 0.5\n    b1 = np.random.rand(10, 1) - 0.5\n    W2 = np.random.rand(10, 10) - 0.5\n    b2 = np.random.rand(10, 1) - 0.5\n    return W1, b1, W2, b2\n","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.025097Z","iopub.execute_input":"2023-04-13T18:01:10.025414Z","iopub.status.idle":"2023-04-13T18:01:10.033540Z","shell.execute_reply.started":"2023-04-13T18:01:10.025387Z","shell.execute_reply":"2023-04-13T18:01:10.032536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def relu(z):\n    return np.maximum(z,0)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.034992Z","iopub.execute_input":"2023-04-13T18:01:10.035456Z","iopub.status.idle":"2023-04-13T18:01:10.043273Z","shell.execute_reply.started":"2023-04-13T18:01:10.035413Z","shell.execute_reply":"2023-04-13T18:01:10.042346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(z):\n    exp = np.exp(z - np.max(z))\n    return exp / exp.sum(axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.044818Z","iopub.execute_input":"2023-04-13T18:01:10.045165Z","iopub.status.idle":"2023-04-13T18:01:10.053865Z","shell.execute_reply.started":"2023-04-13T18:01:10.045121Z","shell.execute_reply":"2023-04-13T18:01:10.052718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def forward_propagation(w1,b1,w2,b2,x):\n    z1=w1.dot(x)+b1 # feed x (og data) into first layer\n    a1=relu(z1) #activation function for first layer\n    z2=w2.dot(a1)+b2 #passing the output og first layer into second layer\n    a2=softmax(z2) # calculating probabilities of output of second layer using softmax\n    return z1,a1,z2,a2","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.055076Z","iopub.execute_input":"2023-04-13T18:01:10.055393Z","iopub.status.idle":"2023-04-13T18:01:10.064778Z","shell.execute_reply.started":"2023-04-13T18:01:10.055366Z","shell.execute_reply":"2023-04-13T18:01:10.063912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a = np.array([1, 0, 3])\n# oh=pd.get_dummies(a)\n# oh=np.array(oh)\n# oh\n\ndef OHE(y):\n    one_hot_y=np.zeros((y.size,y.max()+1))\n    one_hot_y[np.arange(y.size),y]=1\n    one_hot_y=one_hot_y.T # we transpose it because we want each column to be an example (linear algerba rules we want Zi and Y to be in the same format)\n    return one_hot_y\n# y=np.array([1, 0, 3])\n# OHE(y)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.067180Z","iopub.execute_input":"2023-04-13T18:01:10.067573Z","iopub.status.idle":"2023-04-13T18:01:10.075842Z","shell.execute_reply.started":"2023-04-13T18:01:10.067531Z","shell.execute_reply":"2023-04-13T18:01:10.074962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a = np.array([1, 0, 3])\n# zerosmat=np.zeros((y.size,y.max()+1))\n# range=np.arange(y.size)\n# oh=pd.get_dummies(a)\n# oh=np.array(oh)\n# range ,zerosmat ,y.max()+1","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.077128Z","iopub.execute_input":"2023-04-13T18:01:10.077497Z","iopub.status.idle":"2023-04-13T18:01:10.085605Z","shell.execute_reply.started":"2023-04-13T18:01:10.077451Z","shell.execute_reply":"2023-04-13T18:01:10.084956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def relu_derivative(Z):\n    return Z > 0\n\ndef back_propgation(z1,a1,a2,w2,x,y):\n    OHE_y=OHE(y)\n    \n    dz2=2*(a2-OHE_y) # 10,784\n    dw2=(1/m)*(dz2.dot(a1.T)) #10,10\n    db2 = 1 / m * np.sum(dz2) #10,1\n\n    \n    dz1=(w2.T).dot(dz2)*(relu_derivative(z1)) #10,m\n    dw1=(1/m)*(dz1.dot(x.T)) #10,784\n    db1 = 1 / m * np.sum(dz1) #10,1\n    \n    return dw1,db1,dw2,db2","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.086850Z","iopub.execute_input":"2023-04-13T18:01:10.087976Z","iopub.status.idle":"2023-04-13T18:01:10.099613Z","shell.execute_reply.started":"2023-04-13T18:01:10.087946Z","shell.execute_reply":"2023-04-13T18:01:10.098724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def update_parameters(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha):\n    W1 = W1 - alpha * dW1\n    b1 = b1 - alpha * db1    \n    W2 = W2 - alpha * dW2  \n    b2 = b2 - alpha * db2    \n    return W1, b1, W2, b2","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.100702Z","iopub.execute_input":"2023-04-13T18:01:10.101916Z","iopub.status.idle":"2023-04-13T18:01:10.110914Z","shell.execute_reply.started":"2023-04-13T18:01:10.101779Z","shell.execute_reply":"2023-04-13T18:01:10.109485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_predictions(A2):\n    return np.argmax(A2, 0)\n\ndef get_accuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y) / Y.size\n\ndef gradient_descent(X, Y, alpha, iterations):\n    W1, b1, W2, b2 = init_params()\n    for i in range(iterations):\n        Z1, A1, Z2, A2 = forward_propagation(W1, b1, W2, b2, x_train)\n        dW1, db1, dW2, db2 = back_propgation(Z1, A1, A2, W2, x_train, y_train)\n        W1, b1, W2, b2 = update_parameters(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha)\n        if i % 10 == 0:\n            print(\"Iteration: \", i)\n            predictions = get_predictions(A2)\n            print(get_accuracy(predictions, Y))\n    return W1, b1, W2, b2","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:01:10.112857Z","iopub.execute_input":"2023-04-13T18:01:10.113377Z","iopub.status.idle":"2023-04-13T18:01:10.123025Z","shell.execute_reply.started":"2023-04-13T18:01:10.113291Z","shell.execute_reply":"2023-04-13T18:01:10.121925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1, b1, W2, b2 = gradient_descent(x_train, y_train, 0.15, 600)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:03:21.204936Z","iopub.execute_input":"2023-04-13T18:03:21.205826Z","iopub.status.idle":"2023-04-13T18:04:06.052011Z","shell.execute_reply.started":"2023-04-13T18:03:21.205788Z","shell.execute_reply":"2023-04-13T18:04:06.051186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this part of the notebook was taken from somewhere else\nimport matplotlib.pyplot as plt\ndef make_predictions(X, W1, b1, W2, b2):\n    _, _, _, A2 = forward_propagation(W1, b1, W2, b2, X)\n    predictions = get_predictions(A2)\n    return predictions\n\ndef test_prediction(index, W1, b1, W2, b2):\n    current_image = x_batch[:, index, None]\n    prediction = make_predictions(x_batch[:, index, None], W1, b1, W2, b2)\n    label = y_batch[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n    \n    current_image = current_image.reshape((28, 28)) * 255\n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:04:06.053685Z","iopub.execute_input":"2023-04-13T18:04:06.054380Z","iopub.status.idle":"2023-04-13T18:04:06.061665Z","shell.execute_reply.started":"2023-04-13T18:04:06.054348Z","shell.execute_reply":"2023-04-13T18:04:06.060934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_prediction(0, W1, b1, W2, b2)\ntest_prediction(1, W1, b1, W2, b2)\ntest_prediction(2, W1, b1, W2, b2)\ntest_prediction(3, W1, b1, W2, b2)","metadata":{"execution":{"iopub.status.busy":"2023-04-13T18:04:06.062996Z","iopub.execute_input":"2023-04-13T18:04:06.063528Z","iopub.status.idle":"2023-04-13T18:04:06.721176Z","shell.execute_reply.started":"2023-04-13T18:04:06.063497Z","shell.execute_reply":"2023-04-13T18:04:06.720339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}