{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nfrom torch import optim\nfrom torch.utils.data import Dataset, DataLoader\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:42.592990Z","iopub.execute_input":"2025-10-27T04:49:42.593753Z","iopub.status.idle":"2025-10-27T04:49:48.087906Z","shell.execute_reply.started":"2025-10-27T04:49:42.593725Z","shell.execute_reply":"2025-10-27T04:49:48.086850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:48.089531Z","iopub.execute_input":"2025-10-27T04:49:48.090150Z","iopub.status.idle":"2025-10-27T04:49:51.690692Z","shell.execute_reply.started":"2025-10-27T04:49:48.090116Z","shell.execute_reply":"2025-10-27T04:49:51.689732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:51.691588Z","iopub.execute_input":"2025-10-27T04:49:51.691948Z","iopub.status.idle":"2025-10-27T04:49:51.700240Z","shell.execute_reply.started":"2025-10-27T04:49:51.691919Z","shell.execute_reply":"2025-10-27T04:49:51.699045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:51.702122Z","iopub.execute_input":"2025-10-27T04:49:51.702376Z","iopub.status.idle":"2025-10-27T04:49:52.546268Z","shell.execute_reply.started":"2025-10-27T04:49:51.702357Z","shell.execute_reply":"2025-10-27T04:49:52.545255Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df, test_df = train_test_split(data, train_size=0.8, random_state=42)\ntrain_df = train_df.to_numpy()\ntest_df = test_df.to_numpy()\ny_train, x_train = train_df[:, 0], train_df[:, 1:].astype(np.float32)\ny_test, x_test = test_df[:, 0], test_df[:, 1:].astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:52.547162Z","iopub.execute_input":"2025-10-27T04:49:52.547531Z","iopub.status.idle":"2025-10-27T04:49:52.900794Z","shell.execute_reply.started":"2025-10-27T04:49:52.547512Z","shell.execute_reply":"2025-10-27T04:49:52.899685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train, y_train, x_test, y_test = map(torch.tensor, (x_train, y_train, x_test, y_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:52.901884Z","iopub.execute_input":"2025-10-27T04:49:52.902202Z","iopub.status.idle":"2025-10-27T04:49:53.053307Z","shell.execute_reply.started":"2025-10-27T04:49:52.902174Z","shell.execute_reply":"2025-10-27T04:49:53.051967Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train.size()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.054238Z","iopub.execute_input":"2025-10-27T04:49:53.054791Z","iopub.status.idle":"2025-10-27T04:49:53.065870Z","shell.execute_reply.started":"2025-10-27T04:49:53.054765Z","shell.execute_reply":"2025-10-27T04:49:53.064974Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eps = 1e-6\ndef relu(x):\n    return x.clamp(0)\n\n\ndef batch_norm(x, gamma):\n    batch_mean = x.mean(dim=0)\n    batch_var = (x - batch_mean).var(dim=0)\n    x_norm = (x - batch_mean) / torch.sqrt(batch_var + eps)\n    return gamma * x_norm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.066769Z","iopub.execute_input":"2025-10-27T04:49:53.067036Z","iopub.status.idle":"2025-10-27T04:49:53.085162Z","shell.execute_reply.started":"2025-10-27T04:49:53.067009Z","shell.execute_reply":"2025-10-27T04:49:53.084180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BATCH_SIZE = 64\nepochs = 300\nEPOCHS_PER_EVAL = 10\nlr = 1e-2\nhidden_features = 512","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.086138Z","iopub.execute_input":"2025-10-27T04:49:53.086457Z","iopub.status.idle":"2025-10-27T04:49:53.105045Z","shell.execute_reply.started":"2025-10-27T04:49:53.086430Z","shell.execute_reply":"2025-10-27T04:49:53.103965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train, y_train, x_test, y_test = [t.to(device) for t in [x_train, y_train, x_test, y_test]]\nw1 = torch.distributions.Uniform(-np.sqrt(1/784), np.sqrt(1/784)).sample((hidden_features, 784)).to(device).requires_grad_()\nb1 = torch.distributions.Uniform(-np.sqrt(1/784), np.sqrt(1/784)).sample((hidden_features, )).to(device).requires_grad_()\nw2 = torch.distributions.Uniform(-np.sqrt(1/hidden_features), np.sqrt(1/hidden_features)).sample((hidden_features, hidden_features)).to(device).requires_grad_()\nb2 = torch.distributions.Uniform(-np.sqrt(1/hidden_features), np.sqrt(1/hidden_features)).sample((hidden_features, )).to(device).requires_grad_()\nw3 = torch.distributions.Uniform(-np.sqrt(1/hidden_features), np.sqrt(1/hidden_features)).sample((10, hidden_features)).to(device).requires_grad_()\nb3 = torch.distributions.Uniform(-np.sqrt(1/hidden_features), np.sqrt(1/hidden_features)).sample((10, )).to(device).requires_grad_()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.108663Z","iopub.execute_input":"2025-10-27T04:49:53.109246Z","iopub.status.idle":"2025-10-27T04:49:53.185329Z","shell.execute_reply.started":"2025-10-27T04:49:53.109223Z","shell.execute_reply":"2025-10-27T04:49:53.184256Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MNIST(Dataset):\n    def __init__(self, x, y):\n        self.x = x\n        self.y = y\n        self. n = len(x)\n    def __len__(self) :\n        return self.n\n    def __getitem__(self, idx):\n        return self.x[idx], self.y[idx]\n\ntrain_ds = MNIST(x_train, y_train)\ntest_ds = MNIST(x_test, y_test)\ntrain_dl = DataLoader(train_ds, batch_size=BATCH_SIZE)\ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.186392Z","iopub.execute_input":"2025-10-27T04:49:53.186692Z","iopub.status.idle":"2025-10-27T04:49:53.193099Z","shell.execute_reply.started":"2025-10-27T04:49:53.186671Z","shell.execute_reply":"2025-10-27T04:49:53.192132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def softmax(x):\n    x_exp = torch.exp(x - x.max(dim=1, keepdim=True).values)\n    return x_exp / x_exp.sum(dim=1, keepdim=True)\n\n\ndef forward(x):\n    x = x / 255.0\n    h1 = relu(x @ w1.T + b1)\n    h1 = batch_norm(h1, gamma=1.0)\n    h2 = relu(h1 @ w2.T + b2)\n    h2 = batch_norm(h2, gamma=1.0)\n    logits = h2 @ w3.T + b3\n    return logits\n\n\ndef cross_entropy_loss(logits, y):\n    probs = softmax(logits)\n    log_likelihood = -torch.log(probs[range(len(y)), y] + eps)\n    return log_likelihood.mean()\n\n\ndef accuracy(x, y):\n    with torch.no_grad():\n        logits = forward(x)\n        preds = torch.argmax(logits, dim=1)\n        return (preds == y).float().mean().item()\n\n\nparams = [w1, b1, w2, b2, w3, b3]\noptimizer = optim.SGD(params, lr=lr)\n\nfor epoch in range(1, epochs + 1):\n    total_loss = 0\n    for xb, yb in train_dl:\n        xb, yb = xb.to(device), yb.to(device)\n\n        logits = forward(xb)\n        loss = cross_entropy_loss(logits, yb)\n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n\n    avg_loss = total_loss / len(train_dl)\n    acc = accuracy(x_test, y_test)\n    print(f\"Epoch [{epoch:03d}/{epochs}] | Loss: {avg_loss:.4f} | Test Accuracy: {acc*100:.2f}%\")\n\nprint(\"\\nFinal Custom MLP Test Accuracy: {:.2f}%\".format(accuracy(x_test, y_test) * 100))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T04:49:53.193642Z","iopub.execute_input":"2025-10-27T04:49:53.193857Z","iopub.status.idle":"2025-10-27T05:09:43.727727Z","shell.execute_reply.started":"2025-10-27T04:49:53.193839Z","shell.execute_reply":"2025-10-27T05:09:43.726680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Built-in MLP\n\nclass TorchMLP(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Linear(784, hidden_features),\n            nn.ReLU(),\n            nn.BatchNorm1d(hidden_features),\n            nn.Linear(hidden_features, hidden_features),\n            nn.ReLU(),\n            nn.BatchNorm1d(hidden_features),\n            nn.Linear(hidden_features, 10)\n        )\n\n    def forward(self, x):\n        return self.net(x / 255.0)\n\nmodel = TorchMLP().to(device)\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=lr)\n\nfor epoch in range(1, epochs + 1):\n    total_loss = 0\n    for xb, yb in train_dl:\n        xb, yb = xb.to(device), yb.to(device)\n        optimizer.zero_grad()\n        out = model(xb)\n        loss = criterion(out, yb)\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n\n    if epoch % EPOCHS_PER_EVAL == 0:\n        with torch.no_grad():\n            preds = torch.argmax(model(x_test), dim=1)\n            acc = (preds == y_test).float().mean().item()\n        print(f\"[Built-in] Epoch {epoch}/{epochs} | Loss: {total_loss/len(train_dl):.4f} | Test Acc: {acc:.4f}\")\n\n\nwith torch.no_grad():\n    preds = torch.argmax(model(x_test), dim=1)\n    built_in_acc = (preds == y_test).float().mean().item()\nprint(f\"\\nFinal Built-in MLP Test Accuracy: {built_in_acc:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-27T05:24:39.760248Z","iopub.execute_input":"2025-10-27T05:24:39.760574Z","iopub.status.idle":"2025-10-27T05:37:08.110439Z","shell.execute_reply.started":"2025-10-27T05:24:39.760553Z","shell.execute_reply":"2025-10-27T05:37:08.109627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Final Observation\nThe Results of the two methods are almost identical. The values are very close for each iteration","metadata":{}}]}