{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom matplotlib import pyplot as plt \n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-17T20:49:36.599256Z","iopub.execute_input":"2023-03-17T20:49:36.600166Z","iopub.status.idle":"2023-03-17T20:49:36.605077Z","shell.execute_reply.started":"2023-03-17T20:49:36.600122Z","shell.execute_reply":"2023-03-17T20:49:36.604058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:36.632133Z","iopub.execute_input":"2023-03-17T20:49:36.633423Z","iopub.status.idle":"2023-03-17T20:49:39.038587Z","shell.execute_reply.started":"2023-03-17T20:49:36.633354Z","shell.execute_reply":"2023-03-17T20:49:39.037212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pandas, reading the csv file in which the data resides.","metadata":{}},{"cell_type":"code","source":"data.head().T\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.04354Z","iopub.execute_input":"2023-03-17T20:49:39.044748Z","iopub.status.idle":"2023-03-17T20:49:39.059288Z","shell.execute_reply.started":"2023-03-17T20:49:39.044691Z","shell.execute_reply":"2023-03-17T20:49:39.057767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\nThings that has to be done to this data:\n1) The rows signify a test example; we want it to be colums (Take transpose of the matrix)\n2) Label is also taking a row in the transpose matrix, set it aside. \n    So, Y_Test becomes the 0th row, containing answers of all the test data\n    The rest rows[1:n] becomes thetest cases; look above. \n3) The pixel range is in the form 0-255; so we divide all the data so it fits in a range of    0-1, making it easier for the system.\n4) Leave some for the test, rest training. \n    \n","metadata":{}},{"cell_type":"code","source":"#Getting the data set ready. \n\ndata = np.array(data) #Converts to an array.\nm, n = data.shape #Gives the number of rows and columns. \nnp.random.shuffle(data)\n\nTest_Data = data[0:1000].T #Each column is an example. \nY_Test = Test_Data[0]\nX_Test = Test_Data[1:n]\nX_Test = X_Test/255\n\nTrain_Data = data[1000:m].T #Each column is an example. \nY_Train = Train_Data[0]\nX_Train = Train_Data[1:n]\nX_Train = X_Train/255\n_, m_train = X_Train.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.060998Z","iopub.execute_input":"2023-03-17T20:49:39.061418Z","iopub.status.idle":"2023-03-17T20:49:39.849896Z","shell.execute_reply.started":"2023-03-17T20:49:39.061381Z","shell.execute_reply":"2023-03-17T20:49:39.848668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def initiate_parameters():\n    W1 = np.random.rand(10, 784) * np.sqrt(1 / 784)\n    W2 = np.random.rand(10, 10) * np.sqrt(1 / 10)\n    b1 = np.zeros((10, 1))\n    b2 = np.zeros((10, 1))\n\n    \n    return W1,b1,W2,b2\n\ndef relu(Z):\n    return np.maximum(0, Z)\n\n\ndef derivative_RelU(Z):\n    return Z > 0\n\n\ndef softmax(Z):\n    exp = np.exp(Z - np.max(Z)) \n    return exp / exp.sum(axis=0)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.852587Z","iopub.execute_input":"2023-03-17T20:49:39.852946Z","iopub.status.idle":"2023-03-17T20:49:39.86137Z","shell.execute_reply.started":"2023-03-17T20:49:39.852914Z","shell.execute_reply":"2023-03-17T20:49:39.86002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, here.\nWe initiate parameters to the matrix with required dimensions. \nRandom numbers, obviously. \nFirst has 10 rows and 784 columns; so, 784 pixels connecting each to the next 10 activation layers. \n","metadata":{}},{"cell_type":"code","source":"def forward_propagation(W1, B1, W2, B2, X):\n    Z1 = W1.dot(X) + B1 #A1 is defined by activation and baises in the layer behind it.\n    A1 = relu(Z1) #To keep the data a little consistent; making the results accutrate.\n    Z2 = W2.dot(A1) + B2 \n    A2 = softmax(Z2) #Squishin values between 0 and 1, making the output consistent.\n    return Z1, A1,Z2, A2 ","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.863097Z","iopub.execute_input":"2023-03-17T20:49:39.863581Z","iopub.status.idle":"2023-03-17T20:49:39.87371Z","shell.execute_reply.started":"2023-03-17T20:49:39.86353Z","shell.execute_reply":"2023-03-17T20:49:39.872182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"AL1= sigmoid(a1W1 + a2W2 +..... + B1) \nAL1 = Relu(W1 dot X (matrix multiplication) + b1)\n","metadata":{}},{"cell_type":"code","source":"def solution_matrix(Y):\n    one_hot_Y = np.zeros((Y.max()+1,Y.size))\n    one_hot_Y[Y,np.arange(Y.size)] = 1 \n    return one_hot_Y\n\ndef back_propagation(Z1,A1,Z2, A2, W1, W2, X, Y):\n    Y1= solution_matrix(Y)\n    dZ2 = 2*(A2- Y1)\n    dW2 = 1/m * (dZ2.dot(A1.T))\n    dB2 = 1/m * (np.sum(dZ2,1))\n    \n    dZ1 = W2.T.dot(dZ2) * derivative_RelU(Z1)\n    dW1 = 1/m * (dZ1.dot(X.T)) \n    dB1 = 1/m * np.sum(dZ1,1)\n    return dW1, dB1, dW2, dB2\n\ndef update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha):\n    W1 -= alpha * dW1\n    b1 -= alpha * np.reshape(db1, (10,1))\n    W2 -= alpha * dW2\n    b2 -= alpha * np.reshape(db2, (10,1))\n\n    return W1, b1, W2, b2   \n\n  \n    ","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.875412Z","iopub.execute_input":"2023-03-17T20:49:39.875786Z","iopub.status.idle":"2023-03-17T20:49:39.887379Z","shell.execute_reply.started":"2023-03-17T20:49:39.875751Z","shell.execute_reply":"2023-03-17T20:49:39.88624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* This is where the magic happens; \n* Intuitively; the change (error) in Z2 depends on the difference in the actual label vs the recieved activation layer; which is the output of the cost function; so a derivative of Z2 on (A2 - Y1)^2 would give us 2(A2-Y1) more or less.\n* And what does the W2 depend on? It depends on the change in Z2 and the activation of the previous layer. \n\n* Then, we upscale by some alpha and apply the changes to the weights and baises.","metadata":{}},{"cell_type":"code","source":"def get_predictions(A2):\n    return np.argmax(A2, 0)\n\ndef get_accuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y) / Y.size\n\ndef gradient_descent(X, Y, alpha, iterations):\n    prediction = 0\n    W1, b1, W2, b2 = initiate_parameters()\n    for i in range(iterations):\n        Z1, A1, Z2, A2 = forward_propagation(W1, b1, W2, b2, X)\n        dW1, db1, dW2, db2 = back_propagation(Z1, A1, Z2, A2, W1, W2, X, Y)\n        W1, b1, W2, b2 = update_params(W1, b1, W2, b2, dW1, db1, dW2, db2, alpha)\n        prediction = get_predictions(A2)\n    print(f'{get_accuracy(prediction, Y):.3%}')\n    return W1, b1, W2, b2\n\nW1, b1, W2, b2 = gradient_descent(X_Train, Y_Train, 0.10, 500)","metadata":{"execution":{"iopub.status.busy":"2023-03-17T20:49:39.88885Z","iopub.execute_input":"2023-03-17T20:49:39.889218Z","iopub.status.idle":"2023-03-17T20:50:57.740764Z","shell.execute_reply.started":"2023-03-17T20:49:39.889182Z","shell.execute_reply":"2023-03-17T20:50:57.739032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get the prediction, get the accuracy; apply the algorithm in gradient_descent; why the name? we are finding the local minima of the cost function. ","metadata":{}},{"cell_type":"code","source":"def make_predictions(X, W1, b1, W2, b2):\n    _, _, _, A2 = forward_propagation(W1, b1, W2, b2, X)\n    predictions = get_predictions(A2)\n    return predictions\n\ndef test_prediction(index, W1, b1, W2, b2):\n    current_image = X_Train[:, index, None]\n    prediction = make_predictions(X_Train[:, index, None], W1, b1, W2, b2)\n    label = Y_Train[index]\n    print(\"Prediction: \", prediction)\n    print(\"Label: \", label)\n    \n    current_image = current_image.reshape((28, 28)) \n    plt.gray()\n    plt.imshow(current_image, interpolation='nearest')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-17T21:03:47.363094Z","iopub.execute_input":"2023-03-17T21:03:47.363504Z","iopub.status.idle":"2023-03-17T21:03:47.372646Z","shell.execute_reply.started":"2023-03-17T21:03:47.363469Z","shell.execute_reply":"2023-03-17T21:03:47.371112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(11, 16):\n    test_prediction(i, W1, b1, W2, b2)\n    \n    \n\n","metadata":{"execution":{"iopub.status.busy":"2023-03-17T21:05:18.463021Z","iopub.execute_input":"2023-03-17T21:05:18.463438Z","iopub.status.idle":"2023-03-17T21:05:19.887778Z","shell.execute_reply.started":"2023-03-17T21:05:18.463398Z","shell.execute_reply":"2023-03-17T21:05:19.886873Z"},"trusted":true},"execution_count":null,"outputs":[]}]}