{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nfrom matplotlib import pyplot as plt\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T14:07:10.294121Z","iopub.execute_input":"2023-02-12T14:07:10.294577Z","iopub.status.idle":"2023-02-12T14:07:10.300818Z","shell.execute_reply.started":"2023-02-12T14:07:10.294531Z","shell.execute_reply":"2023-02-12T14:07:10.299514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# determine the supported device\ndef get_device():\n    if torch.cuda.is_available():\n        device = torch.device('cuda:0')\n    else:\n        device = torch.device('cpu') # don't have GPU\n    return device\n\n# convert a df to tensor to be used in pytorch\ndef df_to_tensor(df):\n    device = get_device()\n    return torch.from_numpy(df.values).float().to(device)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:10.306637Z","iopub.execute_input":"2023-02-12T14:07:10.307285Z","iopub.status.idle":"2023-02-12T14:07:10.316637Z","shell.execute_reply.started":"2023-02-12T14:07:10.307249Z","shell.execute_reply":"2023-02-12T14:07:10.315639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tensorRowShuffle(X):\n    X=X[torch.randperm(X.size()[0])]\n    return X","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:10.318429Z","iopub.execute_input":"2023-02-12T14:07:10.318770Z","iopub.status.idle":"2023-02-12T14:07:10.331517Z","shell.execute_reply.started":"2023-02-12T14:07:10.318741Z","shell.execute_reply":"2023-02-12T14:07:10.329882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def ReLU(X):\n    return torch.maximum(X,torch.zeros(1))\n\ndef dReLU(X):\n    return X>0\n   \ndef softmax(X):\n    return torch.exp(X)/torch.sum(torch.exp(X), 0)\n\n\n\ndef decodeLabels(Y):\n    Out = torch.zeros(Y.size(dim = 0), int((Y.max() + 1).item()))\n    linTensor = torch.arange(Y.size(dim = 0))\n    \n    linTensor = linTensor.long()\n    Y = Y.long()\n    Out[linTensor, Y] = 1\n    Out = Out.T\n    return Out\n    ","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:10.333451Z","iopub.execute_input":"2023-02-12T14:07:10.334125Z","iopub.status.idle":"2023-02-12T14:07:10.345370Z","shell.execute_reply.started":"2023-02-12T14:07:10.334085Z","shell.execute_reply":"2023-02-12T14:07:10.344170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfData = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntnData = df_to_tensor(dfData)\ntnData = tensorRowShuffle(tnData)\n\nn = tnData.size(dim = 0)\npxPerImg = tnData.size(dim = 1)\nnTestImages = 1000\n\ntsSet = tnData[0:nTestImages].T\nX_tsSet = tsSet[1:pxPerImg] / 255\nY_tsSet = tsSet[0]\n\ntrSet = tnData[nTestImages:n].T\nX_trSet = trSet[1:pxPerImg] / 255\nY_trSet = trSet[0]\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:10.347218Z","iopub.execute_input":"2023-02-12T14:07:10.347584Z","iopub.status.idle":"2023-02-12T14:07:13.640609Z","shell.execute_reply.started":"2023-02-12T14:07:10.347551Z","shell.execute_reply":"2023-02-12T14:07:13.639645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def getPredictions(A):\n    return np.argmax(A, 0)\n\ndef getAccuracy(predictions, Y):\n    print(predictions, Y)\n    return np.sum(predictions == Y) / Y.size","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:13.641880Z","iopub.execute_input":"2023-02-12T14:07:13.642621Z","iopub.status.idle":"2023-02-12T14:07:13.648918Z","shell.execute_reply.started":"2023-02-12T14:07:13.642569Z","shell.execute_reply":"2023-02-12T14:07:13.647533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def initParams():\n    W1 = torch.rand((10,784)) - 0.5\n    W2 = torch.rand((10,10)) - 0.5\n    W3 = torch.rand((10,10)) - 0.5\n\n    b1 = torch.rand((10,1)) - 0.5\n    b2 = torch.rand((10,1)) - 0.5\n    b3 = torch.rand((10,1)) - 0.5\n    \n    return W1, W2, W3, b1, b2, b3\n\ndef forwardProp(W1, W2, W3, b1, b2, b3, X):\n    A0 = X\n\n    Z1 = torch.mm(W1, A0) + b1\n    A1 = ReLU(Z1)\n    Z2 = torch.mm(W2, A1) + b2\n    A2 = ReLU(Z2)\n    Z3 = torch.mm(W3, A2) + b3\n    A3 = softmax(Z3)\n    \n    return A1, A2, A3, Z1, Z2, Z3\n\ndef backwardProp(X,Y, A1, A2, A3, Z1, Z2, Z3, W2, W3):\n    k = 1/Y.size(dim = 1)\n    \n    dZ3 = A3 - Y\n    dW3 = k * torch.mm(dZ3, A2.T)\n    db3 = k * torch.sum(dZ3, dim=1)\n    \n    dZ2 = torch.mm(W3.T, dZ3) * dReLU(Z2)\n    dW2 = k * torch.mm(dZ2, A1.T)\n    db2 = k * torch.sum(dZ2, dim=1)\n    \n    dZ1 = torch.mm(W2.T, dZ2) * dReLU(Z1)\n    dW1 = k * torch.mm(dZ1, X.T)\n    db1 = k * torch.sum(dZ1, dim=1) \n    \n    return dW1, dW2, dW3, db1, db2, db3\n    \ndef paramCorrection(W1, W2, W3, b1, b2, b3, dW1, dW2, dW3, db1, db2, db3, alpha):\n    nW1 = W1 - alpha * dW1\n    nW2 = W2 - alpha * dW2\n    nW3 = W3 - alpha * dW3\n    nb1 = b1 - alpha * torch.reshape(db1, (10, 1))\n    nb2 = b2 - alpha * torch.reshape(db2, (10, 1))\n    nb3 = b3 - alpha * torch.reshape(db3, (10, 1))\n    \n    return nW1, nW2, nW3, nb1, nb2, nb3","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:13.652052Z","iopub.execute_input":"2023-02-12T14:07:13.652863Z","iopub.status.idle":"2023-02-12T14:07:13.671198Z","shell.execute_reply.started":"2023-02-12T14:07:13.652815Z","shell.execute_reply":"2023-02-12T14:07:13.669929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def gradientDescent(X, Y, iterations, alpha):\n    \n    W1, W2, W3, b1, b2, b3 = initParams()\n    Ymat = decodeLabels(Y)\n    \n    for i in range(iterations):\n        A1, A2, A3, Z1, Z2, Z3 = forwardProp(W1, W2, W3, b1, b2, b3, X)\n        dW1, dW2, dW3, db1, db2, db3 = backwardProp(X, Ymat, A1, A2, A3, Z1, Z2, Z3, W2, W3)\n        W1, W2, W3, b1, b2, b3 = paramCorrection(W1, W2, W3, b1, b2, b3, dW1, dW2, dW3, db1, db2, db3, alpha)\n        if (i%10 == 0):\n            print('iteration:', i)\n            print('accuracy:', getAccuracy(getPredictions(A3.numpy()), Y.numpy()) ,'\\n')\n        \n    return W1, W2, W3, b1, b2, b3","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:07:13.672758Z","iopub.execute_input":"2023-02-12T14:07:13.673215Z","iopub.status.idle":"2023-02-12T14:07:13.686192Z","shell.execute_reply.started":"2023-02-12T14:07:13.673182Z","shell.execute_reply":"2023-02-12T14:07:13.684801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"W1, W2, W3, b1, b2, b3 = gradientDescent(X_trSet, Y_trSet, 1000, 0.1)\n\n","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:08:19.916769Z","iopub.execute_input":"2023-02-12T14:08:19.918135Z","iopub.status.idle":"2023-02-12T14:09:01.448729Z","shell.execute_reply.started":"2023-02-12T14:08:19.918079Z","shell.execute_reply":"2023-02-12T14:09:01.447547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def testImg(W1, W2, W3, b1, b2, b3, img, lbl):\n   \n    img = img[:, None]\n    A1, A2, A3, Z1, Z2, Z3 = forwardProp(W1, W2, W3, b1, b2, b3, img)\n    \n    cat = torch.argmax(A3)\n    \n    vImg = np.reshape(img, (28,28))*255\n    \n    plt.gray()\n    plt.imshow(vImg, interpolation='nearest')\n    plt.show()\n    \n    \n    print('Estimated Number:', cat.item())\n    print('Correct Number:', int(lbl.item()))\n    \n    \n    return 0\n\nnTestImgs = 15\n\nimgIndexes = np.random.randint(0, high = 1000, size = nTestImgs) \n\nfor i in range (nTestImgs):\n    testImg(W1, W2, W3, b1, b2, b3, X_tsSet[:,imgIndexes[i]], Y_tsSet[imgIndexes[i]])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T14:25:50.112046Z","iopub.execute_input":"2023-02-12T14:25:50.112485Z","iopub.status.idle":"2023-02-12T14:25:53.157262Z","shell.execute_reply.started":"2023-02-12T14:25:50.112450Z","shell.execute_reply":"2023-02-12T14:25:53.156077Z"},"trusted":true},"execution_count":null,"outputs":[]}]}