{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/digit-recognizer'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-09T05:40:49.571313Z","iopub.execute_input":"2025-05-09T05:40:49.572077Z","iopub.status.idle":"2025-05-09T05:40:49.975737Z","shell.execute_reply.started":"2025-05-09T05:40:49.572038Z","shell.execute_reply":"2025-05-09T05:40:49.974875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf\n# format cols label, pixel0-783 (28 x 28)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T05:40:50.679593Z","iopub.execute_input":"2025-05-09T05:40:50.680392Z","iopub.status.idle":"2025-05-09T05:40:53.495147Z","shell.execute_reply.started":"2025-05-09T05:40:50.680358Z","shell.execute_reply":"2025-05-09T05:40:53.494332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt \nimage_1_X = df.iloc[0][1:].to_numpy()\nimage_1_y = df.iloc[0][0]\n\nprint(\"label\", image_1_y)\n# print(image_1_X)\nimage_1_X = image_1_X.reshape(28, 28)\n# print(image_1_X)\nplt.imshow(image_1_X, cmap='gray')\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nclass linear:\n    def __init__(self, W, b, lr):\n        self.W = W\n        self.b = b\n        self.lr = lr\n\n    def forward(self, x_in):\n        self.x_in = x_in # [num_images x pixels]\n        '''\n        but now we have multiple images\n        z is the result of the linear layer which should which have the preactivation outout for each image\n        so the result should be [num_perceptrons x num_images]\n        which we get from [num_perceptrons, pixels] x [num_images, pixels].T\n        '''\n        self.z = np.matmul(self.W, x_in.T) + self.b\n        # [num_perceptrons, pixels] x [num_images, pixels].T + [num_perceptrons x 1] = [num_perceptrons x num_images]\n        return self.z\n\n    def backward(self, g_out):\n        g_in = np.matmul(g_out.T, self.W) # [num_perceptrons, num_images].T x [num_perceptrons, pixels]\n        dJdW = np.matmul(g_out, self.x_in) # [num_perceptrons, num_images].T x [num_images x pixels]\n        # print(\"dJdb preback\", g_out.shape)\n        dJdb = np.array(np.sum(g_out, axis=1)).reshape(-1, 1) # [num_perceptrons x 1]\n        # print(\"dJdb postback\", dJdb.shape)\n        \n        return g_in, dJdW, dJdb\n\n    def update(self, dJdW, dJdb):\n        self.W -= self.lr * dJdW\n        self.b -= self.lr * dJdb\n\nclass nonlinear:\n    def softmax_forward(self, z):\n        z -= np.max(z, axis = 0) # prevents inf and overflow \n        z = np.exp(z)\n        h = z / np.sum(z, axis = 0)\n        return h\n        \n    def softmax_backward(self, g_out, h): #A lot of implemtations just seem to straight not have a backward for softmax\n        # derivative of softmax\n        g_in = g_out * h\n        return g_in\n    \n    def relu_forward(self, z):\n        # pass through activation \n        h = np.maximum(0, z) \n        return h # [num_perceptrons x num_images]\n\n    def relu_backward(self, g_out, h):\n        # derivative of relu h' = np.where(h >= 0, 1, 0)\n        g_in = g_out * np.where(h >= 0, 1, 0)\n        return g_in","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T05:41:07.626908Z","iopub.execute_input":"2025-05-09T05:41:07.627209Z","iopub.status.idle":"2025-05-09T05:41:07.636327Z","shell.execute_reply.started":"2025-05-09T05:41:07.627185Z","shell.execute_reply":"2025-05-09T05:41:07.635308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nX = df.iloc[:,1:] # input data\nX = X/255.0\ny = df.iloc[0:,0] # correct labels\n# print(\"y\", y)\n\nW1 = np.random.randn(128, X.shape[1]) * np.sqrt(2 / X.shape[1])\nb1 = np.random.randn(128, 1)\nprint(\"X:\", X.shape)\nprint(\"W1:\", W1.shape)\nprint(\"b1:\", b1.shape)\n\nW2 = np.random.randn(10, 128) * np.sqrt(2 / 128)\nb2 = np.random.randn(10, 1)\nprint(\"W2:\", W2.shape)\nprint(\"b2:\", b2.shape)\nlr = 0.0001\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T06:07:52.149651Z","iopub.execute_input":"2025-05-09T06:07:52.150492Z","iopub.status.idle":"2025-05-09T06:07:54.623490Z","shell.execute_reply.started":"2025-05-09T06:07:52.150418Z","shell.execute_reply":"2025-05-09T06:07:54.622518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lin_layer_1 = linear(W1, b1, lr)\nrelu_layer_1 = nonlinear()\n\nlin_layer_2 = linear(W2, b2, lr)\nsoftmax_final_layer = nonlinear()\n\nbatch_size = 1000\n\nnum_samples = X.shape[0]  # Total number of samples\nnum_batches = num_samples // batch_size  # Number of full batches\n\nnum_epochs = 10 \nfor epoch in range(num_epochs):\n    for batch_idx in range(num_batches):\n        # Extract batch data\n        start = batch_idx * batch_size\n        end = start + batch_size\n        X_batch = X[start:end]\n        y_batch = y[start:end]\n    \n        # print(f\"batch[{start}:{end}]\")\n        # Forward Pro\n        '''\n        X_batch [50, 784]\n        W1 [16, 784]\n        z1,h1 [16, 50]\n        h1.t = [50, 16]\n        \n        W2 [10, 16]\n        [z1][W2].T = [50, 16] [16, 10] = [50, 10]\n        z2,h2 [50, 10]\n        '''\n        z1 = lin_layer_1.forward(X_batch)\n        h1 = relu_layer_1.relu_forward(z1)\n    \n        z2 = lin_layer_2.forward(h1.T) # transpose so it is in the shape of [num_samples, inputs]\n        h2 = softmax_final_layer.softmax_forward(z2)\n        \n        # onehot y\n        one_hot_y = np.zeros(h2.shape)\n        # print(\"z1\", z1.head(10)) \n        # print(\"z2\", z2.head(10)) \n        for i, l in enumerate(y_batch): # [outputs, samples]\n            one_hot_y[l, i] = 1\n        # print(mat_y)\n        g_out = (h2 - one_hot_y)\n        \n        # Backward propagation\n        \n        # g_in = softmax_final_layer.softmax_backward(g_out, h2)\n        # g_out = g_in\n        \n        g_in, dJdW, dJdb = lin_layer_2.backward(g_out)\n        lin_layer_2.update(dJdW, dJdb)\n        g_out = g_in # g_in for this layer is g_out for the one below it\n        \n        g_in = relu_layer_1.relu_backward(g_out, h1.T)\n        g_out = g_in\n        \n        _, dJdW, dJdb = lin_layer_1.backward(g_out.T)\n        lin_layer_1.update(dJdW, dJdb)\n        \n    if epoch % 1 == 0:\n        preds = np.argmax(h2, axis=0)\n        accuracy = np.mean(preds == y_batch)\n        print(f\"Epoch {epoch + 1}/{num_epochs}, Accuracy: {accuracy:.4f}\")\n\n# image_1_X = np.array(image_1_X).reshape(1, 784)\n# z1 = lin_layer_1.forward(image_1_X)\n# h1 = relu_layer_1.relu_forward(z1)\n# z2 = lin_layer_2.forward(h1.T)\n# h2 = softmax_final_layer.softmax_forward(z2)\n\n# print(z2)\n# print(h2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T06:07:54.624801Z","iopub.execute_input":"2025-05-09T06:07:54.625040Z","iopub.status.idle":"2025-05-09T06:08:07.342814Z","shell.execute_reply.started":"2025-05-09T06:07:54.625021Z","shell.execute_reply":"2025-05-09T06:08:07.341862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n# test_df","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"z1 = lin_layer_1.forward(test_df)\nh1 = relu_layer_1.relu_forward(z1)\nz2 = lin_layer_2.forward(h1.T)\nh2 = softmax_final_layer.softmax_forward(z2)\n\npredictions = np.argmax(h2, axis=0)\nimage_id = [i+1 for i,p in enumerate(predictions)]\n\n\nnp.savetxt(\n    'submission.csv',\n    np.rec.fromarrays([image_id, predictions]),\n    fmt=['%s', '%d'],\n    delimiter=',',\n    header='ImageId,Label',\n    comments='',\n)\n\n!head submission.csv\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"## Why does adding another layer make the program kill itself. Based on the error the values are too high but how?\n# solved with He initialization which is apparenlty good for relu\n\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n\nX = df.iloc[:,1:] # input data\nX = X/255.0\ny = df.iloc[0:,0] # correct labels\n# print(\"y\", y)\n\n\nW1 = np.random.randn(128, X.shape[1]) * np.sqrt(2 / X.shape[1])\nb1 = np.random.randn(128, 1)\nW2 = np.random.randn(64, 128) * np.sqrt(2 / 64)\nb2 = np.random.randn(64, 1)\nW3 = np.random.randn(10, 64) * np.sqrt(2 / 16)\nb3 = np.random.randn(10, 1)\nlr = 0.0001\n\nlin_layer_1 = linear(W1, b1, lr)\nrelu_layer_1 = nonlinear()\n\nlin_layer_2 = linear(W2, b2, lr)\nrelu_layer_2 = nonlinear()\n\nlin_layer_3 = linear(W3, b3, lr)\nsoftmax_final_layer = nonlinear()\n\nprint(\"X:\", X.shape)\nprint(\"W1:\", W1.shape, \"\\tb1:\", b1.shape)\nprint(\"W2:\", W2.shape, \"\\tb2:\", b2.shape)\nprint(\"W3:\", W3.shape, \"\\tb3:\", b3.shape)\n\nbatch_size = 1000\n\nnum_samples = X.shape[0]  # Total number of samples\nnum_batches = num_samples // batch_size  # Number of full batches\n\nnum_epochs = 10 \nfor epoch in range(num_epochs):\n    for batch_idx in range(num_batches):\n        # Extract batch data\n        start = batch_idx * batch_size\n        end = start + batch_size\n        X_batch = X[start:end]\n        y_batch = y[start:end]\n\n        # Forward Pro\n        z1 = lin_layer_1.forward(X_batch) \n        h1 = relu_layer_1.relu_forward(z1)\n    \n        z2 = lin_layer_2.forward(h1.T) # transpose so it is in the shape of [num_samples, inputs]\n        h2 = relu_layer_2.relu_forward(z2)\n\n        z3 = lin_layer_3.forward(h2.T)\n        h3 = softmax_final_layer.softmax_forward(z3)\n        \n        # onehot y\n        one_hot_y = np.zeros(h3.shape)\n        for i, l in enumerate(y_batch): # [outputs, samples]\n            one_hot_y[l, i] = 1\n        g_out = (h3 - one_hot_y)\n        \n        # Backward propagation\n\n        g_in, dJdW, dJdb = lin_layer_3.backward(g_out)\n        lin_layer_3.update(dJdW, dJdb)\n        g_out = g_in\n\n        g_in = relu_layer_2.relu_backward(g_out, h2.T)\n        g_out = g_in\n        g_in, dJdW, dJdb = lin_layer_2.backward(g_out.T)\n        lin_layer_2.update(dJdW, dJdb)\n        g_out = g_in\n        \n        g_in = relu_layer_1.relu_backward(g_out, h1.T)\n        g_out = g_in\n        _, dJdW, dJdb = lin_layer_1.backward(g_out.T)\n        lin_layer_1.update(dJdW, dJdb)\n        \n    if epoch % 1 == 0:\n        preds = np.argmax(h3, axis=0)\n        accuracy = np.mean(preds == y_batch)\n        print(f\"Epoch {epoch + 1}/{num_epochs}, Accuracy: {accuracy:.4f}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-09T06:06:40.278422Z","iopub.execute_input":"2025-05-09T06:06:40.278976Z","iopub.status.idle":"2025-05-09T06:06:58.475511Z","shell.execute_reply.started":"2025-05-09T06:06:40.278953Z","shell.execute_reply":"2025-05-09T06:06:58.474532Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}