{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Set up imports","metadata":{"_uuid":"3bf074cd-2ee4-4ddb-b99a-d8bc47c7dc6d","_cell_guid":"e9640e21-4918-4222-86a7-1baa9ff82edd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\n\n# Visualization tools\nimport torchvision\nimport torchvision.transforms.v2 as transforms\nimport torchvision.transforms.functional as F\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\n\n# Utilities for I/O and file manipulation\nimport numpy as np\nimport pandas as pd\nimport os \nfrom contextlib import redirect_stdout","metadata":{"_uuid":"a8404a12-e32d-478d-b359-d0f91e77c06e","_cell_guid":"166a85b1-0964-4e30-b3c8-40c992ed7054","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:48.253627Z","iopub.execute_input":"2025-03-25T14:08:48.253880Z","iopub.status.idle":"2025-03-25T14:08:54.018565Z","shell.execute_reply.started":"2025-03-25T14:08:48.253853Z","shell.execute_reply":"2025-03-25T14:08:54.017624Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Reproducibility seed and compute device (GPU or CPU)","metadata":{"_uuid":"a4b0436c-d9dc-4fd5-8ae6-ae2e00cc7e7b","_cell_guid":"f7c4260c-93a3-4538-87ef-161d8fdf89bd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Set random seed for reproducibility of all random operations on PyTorch\ntorch.manual_seed(42)\n\n# Set computing device, \"cuda\" stands for an Nvidia GPU \n# Data and models will be sent to this device when training\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"_uuid":"adee5c16-949f-4be4-85a0-0043e7c41076","_cell_guid":"528e22c0-a848-4495-8b8b-7b2becac5008","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:54.020051Z","iopub.execute_input":"2025-03-25T14:08:54.020416Z","iopub.status.idle":"2025-03-25T14:08:54.070445Z","shell.execute_reply.started":"2025-03-25T14:08:54.020386Z","shell.execute_reply":"2025-03-25T14:08:54.069606Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prepare dataset","metadata":{"_uuid":"73b1d80a-85ce-49b3-9f96-2d0ce1c980ca","_cell_guid":"3f770ff4-e1e4-45d1-b5eb-8e390898ba4b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Create an initial transform from PIL to PyTorch tensor to compute mean and standard deviation of the training set\ninitial_transform = transforms.Compose([transforms.ToImage(),\n                                        transforms.ToDtype(torch.float32, scale=True)])","metadata":{"_uuid":"42621d09-9152-4e3f-8ebe-d270eefd2b9b","_cell_guid":"0bd81053-dbb2-40f6-9556-e2078538e6e8","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:54.071477Z","iopub.execute_input":"2025-03-25T14:08:54.071814Z","iopub.status.idle":"2025-03-25T14:08:54.076297Z","shell.execute_reply.started":"2025-03-25T14:08:54.071784Z","shell.execute_reply":"2025-03-25T14:08:54.075224Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define the MNIST Dataset class\nclass MNISTDataset(Dataset):\n    def __init__(self, dataframe, labels=True, transform=None):\n        self.data = dataframe\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        if self.labels:\n            image = self.data.iloc[idx, 1:].values.astype(np.float32).reshape(28, 28)\n            label = self.data.iloc[idx, 0]\n        else:\n            image = self.data.iloc[idx, :].values.astype(np.float32).reshape(28, 28)\n            label = 11  # Placeholder for test data\n\n        if self.transform:\n            image = self.transform(image)\n\n         # Ensure tensors are properly detached and typecast\n        image_tensor = torch.from_numpy(np.array(image)).clone().detach().float()\n        label_tensor = torch.tensor(label).clone().detach().long()\n\n        return image_tensor, label_tensor\n\n# Load the data\ntrain_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data  = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"_uuid":"6da95d28-247e-41e6-ae0d-2628b96f0bfc","_cell_guid":"375b9c61-f669-4fbf-8e98-302b4dde6852","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:54.077350Z","iopub.execute_input":"2025-03-25T14:08:54.077651Z","iopub.status.idle":"2025-03-25T14:08:58.623144Z","shell.execute_reply.started":"2025-03-25T14:08:54.077624Z","shell.execute_reply":"2025-03-25T14:08:58.622406Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head(1)","metadata":{"_uuid":"99abce3d-2885-45f1-b67b-7318b5467b3e","_cell_guid":"125cb50f-c4cd-4580-9721-85a4936b1cf5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:58.625830Z","iopub.execute_input":"2025-03-25T14:08:58.626451Z","iopub.status.idle":"2025-03-25T14:08:58.644681Z","shell.execute_reply.started":"2025-03-25T14:08:58.626390Z","shell.execute_reply":"2025-03-25T14:08:58.643793Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.head(1)","metadata":{"_uuid":"7f654045-0448-407f-82f4-428a0f5d1f35","_cell_guid":"dbf655a9-d0c0-4c7f-9258-1fed48200da2","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:58.645754Z","iopub.execute_input":"2025-03-25T14:08:58.646075Z","iopub.status.idle":"2025-03-25T14:08:58.657201Z","shell.execute_reply.started":"2025-03-25T14:08:58.646047Z","shell.execute_reply":"2025-03-25T14:08:58.656348Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create datasets\ntrain_set = MNISTDataset(train_data, labels=True, transform=initial_transform)\ntest_set = MNISTDataset(test_data, labels=False, transform=initial_transform)","metadata":{"_uuid":"9b9c4494-8980-4917-a36a-0fff199a85a4","_cell_guid":"e0cd1bdc-ef70-4b48-b07e-1d34e6c9c178","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:58.658369Z","iopub.execute_input":"2025-03-25T14:08:58.658716Z","iopub.status.idle":"2025-03-25T14:08:58.670019Z","shell.execute_reply.started":"2025-03-25T14:08:58.658679Z","shell.execute_reply":"2025-03-25T14:08:58.669137Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The images are initially pairs of PIL (Python Image Library) images and their labels\ntrain_set[0]","metadata":{"_uuid":"d33578e5-0c60-4ea6-916b-0f35c5e7e497","_cell_guid":"d202ff94-b38b-4d89-8ad6-b58236f626ac","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:58.671187Z","iopub.execute_input":"2025-03-25T14:08:58.671565Z","iopub.status.idle":"2025-03-25T14:08:58.781549Z","shell.execute_reply.started":"2025-03-25T14:08:58.671528Z","shell.execute_reply":"2025-03-25T14:08:58.780595Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Each image is 28x28 pixels and represented in grayscale\nim = train_set[7][0]\nplt.imshow(im.squeeze(), cmap='gray');","metadata":{"_uuid":"524bb208-c014-48ab-bd40-b6eaf7cef52c","_cell_guid":"8209bd3d-4f48-4188-9ec5-16503e998538","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:58.782685Z","iopub.execute_input":"2025-03-25T14:08:58.783029Z","iopub.status.idle":"2025-03-25T14:08:59.035999Z","shell.execute_reply.started":"2025-03-25T14:08:58.782990Z","shell.execute_reply":"2025-03-25T14:08:59.035053Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We use 0 to represent black pixels and 255 to represent white in uint8 (unsigned integers of 8 bits)\ndf = pd.DataFrame(np.array(im).squeeze())\nplt.figure(figsize=(12, 10))\nsns.heatmap(df, cmap=\"Greys_r\", annot=True, fmt=\".0f\", cbar=False)\nplt.show()","metadata":{"_uuid":"54b31887-fc44-4fc4-aa6c-53a334837190","_cell_guid":"a254da39-a18b-4f23-920f-874f9383ff18","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:08:59.037237Z","iopub.execute_input":"2025-03-25T14:08:59.037562Z","iopub.status.idle":"2025-03-25T14:09:00.612686Z","shell.execute_reply.started":"2025-03-25T14:08:59.037533Z","shell.execute_reply":"2025-03-25T14:09:00.611859Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We can inspect the values used to represent the image by converting it\n# into a Numpy array or PyTorch tensor\narr = np.array(im)\n\narr.shape","metadata":{"_uuid":"099f4d86-94c0-4635-88fe-8c754e197bd2","_cell_guid":"a7c9adaa-15e6-486b-ba5f-f23ac5c4c181","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.613782Z","iopub.execute_input":"2025-03-25T14:09:00.614046Z","iopub.status.idle":"2025-03-25T14:09:00.619905Z","shell.execute_reply.started":"2025-03-25T14:09:00.614019Z","shell.execute_reply":"2025-03-25T14:09:00.618978Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The 6:12 asks for the values in the rows from index 6 (included) to 12 (not included) \n# and the same for the columns. NumPy indexes from top to bottom and left to right.\n# This section is located in the top-left corner of the image\narr[6:12, 6:12]","metadata":{"_uuid":"fa2b7fa5-30c1-4541-bd3e-d06be2771f06","_cell_guid":"f5bc580b-5136-4315-95f8-618821c8fc33","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.620932Z","iopub.execute_input":"2025-03-25T14:09:00.621559Z","iopub.status.idle":"2025-03-25T14:09:00.634527Z","shell.execute_reply.started":"2025-03-25T14:09:00.621531Z","shell.execute_reply":"2025-03-25T14:09:00.633681Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" \n# Here's the same as a PyTorch tensor. \n# We use transforms.ToImage() to convert the PIL image into a PyTorch tensor of unsigned integers\nto_imag = transforms.ToImage()\nmy_tensor = to_imag(im)\n# The shape of the tensor includes the number of channels as a first dimension.\n# Grayscale images have only 1 channel. This is appended in front of the shape tensor\nmy_tensor.shape","metadata":{"_uuid":"d9313276-e502-495a-a728-f6234e502c26","_cell_guid":"c13fc2be-db58-4080-a28e-ca091546ff11","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.635384Z","iopub.execute_input":"2025-03-25T14:09:00.635667Z","iopub.status.idle":"2025-03-25T14:09:00.648751Z","shell.execute_reply.started":"2025-03-25T14:09:00.635641Z","shell.execute_reply":"2025-03-25T14:09:00.647914Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"my_tensor.unsqueeze(0).dim()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.652153Z","iopub.execute_input":"2025-03-25T14:09:00.652391Z","iopub.status.idle":"2025-03-25T14:09:00.666365Z","shell.execute_reply.started":"2025-03-25T14:09:00.652367Z","shell.execute_reply":"2025-03-25T14:09:00.665411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We can remove this dimension by using the squeeze() method, afterwards we can index the values just like we did with the numpy array\nselection_tensor = my_tensor.squeeze()[6:12, 6:12]\nselection_tensor","metadata":{"_uuid":"8bfd613e-7a53-4f0e-8118-74ff9b5b1d75","_cell_guid":"fa31f28e-df6f-4215-86d2-f7f351011e74","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.667399Z","iopub.execute_input":"2025-03-25T14:09:00.667670Z","iopub.status.idle":"2025-03-25T14:09:00.680383Z","shell.execute_reply.started":"2025-03-25T14:09:00.667645Z","shell.execute_reply":"2025-03-25T14:09:00.679583Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Issues with numerical precision","metadata":{"_uuid":"c57fd35f-ca6b-4338-9eeb-954fad258d0a","_cell_guid":"c7087d27-5fb9-4017-b7a4-ce3aecd9e4ac","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# torch.uint8 can only represent numbers in the range of 0 to 255, this causes issues with common functions\n# that we use in neural networks, notice that applying an exponential function to the input will make many values \n# \"explode\" to infinity, this would destroy information during the training of a neural network \n# In this example selection_tensor is implicitly converted to a torch.float32 when passed to the torch.exp function \ntorch.exp(selection_tensor)","metadata":{"_uuid":"749429bd-8832-4616-87ed-434d0266d206","_cell_guid":"ac5aad4d-1677-4a38-9cf4-0a3444048b14","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.681584Z","iopub.execute_input":"2025-03-25T14:09:00.681901Z","iopub.status.idle":"2025-03-25T14:09:00.747271Z","shell.execute_reply.started":"2025-03-25T14:09:00.681864Z","shell.execute_reply":"2025-03-25T14:09:00.746472Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# We can avoid this issue by rescaling the input in the range of 0 to 1 \n# Please take note that this division makes an implicit type conversion to torch.float32\n# In the following section we will see the more common ways that this scaling is done in PyTorch pipelines\ntorch.exp(selection_tensor/255)","metadata":{"_uuid":"ccc4d25a-2d40-4ed0-9997-98be4460f605","_cell_guid":"9b5941b6-5d2c-40be-9232-da4b05c750a5","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.748501Z","iopub.execute_input":"2025-03-25T14:09:00.748859Z","iopub.status.idle":"2025-03-25T14:09:00.755660Z","shell.execute_reply.started":"2025-03-25T14:09:00.748821Z","shell.execute_reply":"2025-03-25T14:09:00.754796Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Moving and transforming tensors","metadata":{"_uuid":"da1525dc-aa08-4ac9-8290-fef84a4165c9","_cell_guid":"7192ab92-7600-43f9-becc-e7c2e7df05fd","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# We can use PyTorch's 'to' function to move tensors to the GPU (if it is available) and change their data type\nfloat_selection_tensor = selection_tensor.to(device, dtype=torch.float32)\nfloat_selection_tensor","metadata":{"_uuid":"e6b9aa6c-f0da-4d08-94ea-2cd7a0e5e049","_cell_guid":"2e81a622-7eb2-4397-8acc-227e364e6d8c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:00.756853Z","iopub.execute_input":"2025-03-25T14:09:00.757190Z","iopub.status.idle":"2025-03-25T14:09:01.138311Z","shell.execute_reply.started":"2025-03-25T14:09:00.757152Z","shell.execute_reply":"2025-03-25T14:09:01.137453Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Another way to transform the data type explicitly is to use transforms.ToDtype, notice that here we are using scale=False\n# This means that we **don't** re-scale the data to the range 0.0 to 1.0\nfloat32_transform = transforms.ToDtype(torch.float32, scale=False)\nfloat_selection_tensor = float32_transform(selection_tensor)\nfloat_selection_tensor","metadata":{"_uuid":"c9d9caed-2077-4374-ab97-dbe5179f6c2a","_cell_guid":"cece1a65-1487-476a-a04c-edade3c2ae54","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:01.139596Z","iopub.execute_input":"2025-03-25T14:09:01.139966Z","iopub.status.idle":"2025-03-25T14:09:01.147730Z","shell.execute_reply.started":"2025-03-25T14:09:01.139924Z","shell.execute_reply":"2025-03-25T14:09:01.146825Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Using scale=True will transform the data into the range of 0 to 1 \nscaling_transform = transforms.ToDtype(torch.float32, scale=True)\nscaling_transform(selection_tensor)","metadata":{"_uuid":"7998cc2c-b41c-4ba6-bd56-17b5de9151e1","_cell_guid":"45498c26-2404-43c9-a738-712e8709285b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:01.148961Z","iopub.execute_input":"2025-03-25T14:09:01.149303Z","iopub.status.idle":"2025-03-25T14:09:01.162284Z","shell.execute_reply.started":"2025-03-25T14:09:01.149262Z","shell.execute_reply":"2025-03-25T14:09:01.161487Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Input normalization with mean and standard deviation of the training set\n\nThe classic paper [Efficient Backprop](https://www.researchgate.net/publication/2811922_Efficient_BackProp) (LeCun et al., 2000) shows empirically that input normalization using the mean and standard deviation of the training set can lead to a 10x improvement in training speed. This practice is widely spread in modern deep learning. Standard scaling transforms the data such that it has a mean of 0 and a standard deviation of 1. This is useful in machine learning algorithms sensitive to the magnitude of features.\n\nWe use the standard scaling normalization (also called [z-score normalization](https://en.wikipedia.org/wiki/Standard_score)):\n\n$x^{normalized} = \\frac{x - \\mu}{\\sigma}$\n\nWhere:\n- $x$ is the input variable\n- $\\mu$ is the mean of inputs across the training set\n- $\\sigma$ is the standard deviation across the training set\n\nIt represents the distance between the raw input x and the population mean in units of the standard deviation. This distance is negative when the raw score is below the mean and positive when above. \n\nImplementation in PyTorch or NumPy:\n```python\nmean = training_data.mean()\nstd = training_data.std()\nnormalized_data = (data - mean) / std\n```\n\nInput normalization is useful for neural networks for these key reasons:\n\n1. Training Stability: Normalized inputs improve gradient flow through the network. Without normalization, large input values can cascade into exploding gradients (values that go outside of the range that we can represent with the number of bits that we have to represent our activations) and unstable training.\n\n2. Faster Learning: Normalization creates a more spherical loss surface, enabling more direct gradient descent paths and allowing higher learning rates.\n\n3. Better Initialization: Most normalized values fall in ranges where activation functions have stronger gradients, preventing early neuron saturation.\n\n**Important**: Always use training set statistics to normalize both training and test data to maintain consistent transformations.","metadata":{"_uuid":"f068646d-d373-483f-b41b-aae89965f628","_cell_guid":"ef7d4bc7-1d9c-49c1-8b2a-9ff880103757","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Calculate mean and std of training set\ntrain_data = torch.stack([initial_transform(img) for img, _ in train_set])\nmean = train_data.mean().item()\nstd = train_data.std()\nprint(f\"Dataset statistics - Mean: {mean:.4f}, Std: {std:.4f}\")","metadata":{"_uuid":"7cacdef6-2eb0-48dd-ad62-a9ac530bf5e3","_cell_guid":"eb352c08-45d2-4ca4-bbb0-e30aad5c5631","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:01.163369Z","iopub.execute_input":"2025-03-25T14:09:01.163654Z","iopub.status.idle":"2025-03-25T14:09:15.779367Z","shell.execute_reply.started":"2025-03-25T14:09:01.163629Z","shell.execute_reply":"2025-03-25T14:09:15.778445Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Now we create a final transformation that will convert the PIL Images to Tensors, scale them, and normalize them\nnorm_transform = transforms.Compose([transforms.ToImage(),\n                                     transforms.ToDtype(torch.float32, scale=True),\n                                     transforms.Normalize((mean,), (std,))                       \n                                    ])","metadata":{"_uuid":"bd751b7f-2a15-4733-a645-ab6b0aed481d","_cell_guid":"6fd24159-5ade-4b93-9950-cb3977d06480","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.780522Z","iopub.execute_input":"2025-03-25T14:09:15.780854Z","iopub.status.idle":"2025-03-25T14:09:15.786000Z","shell.execute_reply.started":"2025-03-25T14:09:15.780824Z","shell.execute_reply":"2025-03-25T14:09:15.784620Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Apply transformation to datasets","metadata":{"_uuid":"a058b408-eecc-477a-a5f6-ba1d9808d7fe","_cell_guid":"9711c496-9120-4508-bcad-47a5617c8448","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# This converts every datapoint in the original dataset to scaled and normalized torch tensor\ntrain_set.transform = norm_transform\ntest_set.transform = norm_transform","metadata":{"_uuid":"99f3b45d-c2dd-463b-9939-fa4ae6341104","_cell_guid":"665e438f-b22d-4e29-9105-1fdb53b3e385","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.786783Z","iopub.execute_input":"2025-03-25T14:09:15.787041Z","iopub.status.idle":"2025-03-25T14:09:15.857985Z","shell.execute_reply.started":"2025-03-25T14:09:15.787012Z","shell.execute_reply":"2025-03-25T14:09:15.857068Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# If we inspect an image from the training_set, we see that it has been re-scaled\ntrain_set[0][0].squeeze()[6:12, 6:12]","metadata":{"_uuid":"092e2d94-85e6-4694-9d69-b696f2ee7216","_cell_guid":"1cdc6884-9ef3-4bf0-9fe3-97953453759c","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.859164Z","iopub.execute_input":"2025-03-25T14:09:15.859923Z","iopub.status.idle":"2025-03-25T14:09:15.877242Z","shell.execute_reply.started":"2025-03-25T14:09:15.859887Z","shell.execute_reply":"2025-03-25T14:09:15.876443Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_set[0][0].min(), train_set[0][0].max()","metadata":{"_uuid":"a6ac0ce0-a623-4eee-8701-23b3dba3a578","_cell_guid":"a3e1c810-fd84-43c8-b2d1-7bd4b4464e27","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.878452Z","iopub.execute_input":"2025-03-25T14:09:15.878698Z","iopub.status.idle":"2025-03-25T14:09:15.892233Z","shell.execute_reply.started":"2025-03-25T14:09:15.878672Z","shell.execute_reply":"2025-03-25T14:09:15.891391Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create a validation set","metadata":{"_uuid":"88c1c42b-e690-410b-9f4a-9498c2df4bb1","_cell_guid":"65b978dd-f91a-4fc5-9dcc-ac3830e6ab95","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Create validation/test split from the test set\nvalidation_size = int(0.25 * len(train_set))  # Split test set in half\ntrain_size = len(train_set) - validation_size\nvalid_subset, train_subset = torch.utils.data.random_split(\n    train_set, [validation_size, train_size], generator=torch.Generator().manual_seed(42)\n)\nlen(valid_subset), len(train_subset)","metadata":{"_uuid":"ed9a110c-8c08-4527-bd94-7548afe24b5c","_cell_guid":"d26ca423-3aad-4efc-856d-677ab08a77ee","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.893361Z","iopub.execute_input":"2025-03-25T14:09:15.893718Z","iopub.status.idle":"2025-03-25T14:09:15.905237Z","shell.execute_reply.started":"2025-03-25T14:09:15.893680Z","shell.execute_reply":"2025-03-25T14:09:15.904490Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Create dataloaders","metadata":{"_uuid":"00ca351a-ce0f-4f1c-81b2-d3db4cec67cf","_cell_guid":"9807eb4c-1f46-4556-9b35-af8c0c89a3cf","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Create data loaders\nbatch_size = 32\n# Notice that we shuffle the training loader, but not the validation or test loaders.\n# This practice of shuffling the training set is one of the techniques that have been shown to improve training.\n# It is described on LeCun's Efficient Backprop publication, but its origin is likely much earlier.\ntrain_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True)\nvalid_loader = DataLoader(valid_subset, batch_size=batch_size)\ntest_loader  = DataLoader(test_set, batch_size=batch_size)","metadata":{"_uuid":"ed05f82b-828d-4c75-9175-fcb37a80acbe","_cell_guid":"2655e8ec-597d-4ec0-9bc5-50077b4cb99f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.906278Z","iopub.execute_input":"2025-03-25T14:09:15.907025Z","iopub.status.idle":"2025-03-25T14:09:15.911983Z","shell.execute_reply.started":"2025-03-25T14:09:15.906985Z","shell.execute_reply":"2025-03-25T14:09:15.911218Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# The batch size is the first dimension of the images tensor. Next comes the number of channels and x, y dimensions.\n# We have 32 labels, one for each datapoint in the batch. \nimages, labels = next(iter(train_loader))\nimages.shape, labels.shape","metadata":{"_uuid":"68de1780-8225-4b77-9754-4146e922ac7a","_cell_guid":"910f62ee-e6c8-4b60-b1ca-1d80212b20fa","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.912905Z","iopub.execute_input":"2025-03-25T14:09:15.913151Z","iopub.status.idle":"2025-03-25T14:09:15.953474Z","shell.execute_reply.started":"2025-03-25T14:09:15.913126Z","shell.execute_reply":"2025-03-25T14:09:15.952695Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Defining a Network","metadata":{"_uuid":"66a17385-226a-4a46-a57f-1c8c5e613ce3","_cell_guid":"90bf53fc-0bf0-4923-a4f7-053b19011202","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Define the model\nmodel = nn.Sequential(\n    nn.Flatten(),\n    nn.Linear(28 * 28, 128),\n    nn.ReLU(),\n    nn.Linear(128, 64),\n    nn.ReLU(),\n    nn.Linear(64, 10)  # Output layer for 10 classes\n)","metadata":{"_uuid":"42257382-45e9-4de3-ae6a-398f92e1d868","_cell_guid":"70b34051-862c-45f5-a626-f9a6bc20faf3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.954477Z","iopub.execute_input":"2025-03-25T14:09:15.954879Z","iopub.status.idle":"2025-03-25T14:09:15.962114Z","shell.execute_reply.started":"2025-03-25T14:09:15.954841Z","shell.execute_reply":"2025-03-25T14:09:15.961263Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Check where the model's weights are \nnext(model.parameters()).device","metadata":{"_uuid":"b0220efa-83a9-4883-bbd6-5179f03c3ccf","_cell_guid":"1fd82e45-ba49-48f3-ad15-2d5ec9299f16","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.963478Z","iopub.execute_input":"2025-03-25T14:09:15.963784Z","iopub.status.idle":"2025-03-25T14:09:15.974273Z","shell.execute_reply.started":"2025-03-25T14:09:15.963756Z","shell.execute_reply":"2025-03-25T14:09:15.973486Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define a loss function and optimizer","metadata":{"_uuid":"344a8a68-b3a9-4ab6-a9bb-fbd7e191abaa","_cell_guid":"a1705c6f-e103-4087-9cf6-a1843f4ab232","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"loss_function = nn.CrossEntropyLoss()","metadata":{"_uuid":"0c70e367-8112-42b3-a57f-d9863919b963","_cell_guid":"1a9e7398-0b96-4247-a22b-e6091eabb731","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.975277Z","iopub.execute_input":"2025-03-25T14:09:15.975550Z","iopub.status.idle":"2025-03-25T14:09:15.988863Z","shell.execute_reply.started":"2025-03-25T14:09:15.975524Z","shell.execute_reply":"2025-03-25T14:09:15.988174Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = Adam(model.parameters())","metadata":{"_uuid":"4d2cb3b6-ea6a-458f-a62e-27010c4da16e","_cell_guid":"65648c2c-1bc7-460f-829f-7a9d5c168235","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:15.989890Z","iopub.execute_input":"2025-03-25T14:09:15.990163Z","iopub.status.idle":"2025-03-25T14:09:15.999594Z","shell.execute_reply.started":"2025-03-25T14:09:15.990138Z","shell.execute_reply":"2025-03-25T14:09:15.998899Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Compute accuracy","metadata":{"_uuid":"fbeab32d-ea85-42f7-8d45-00e815f3ef84","_cell_guid":"11258f1c-0d81-45a9-a6ec-a854dd6f9d04","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"$$\n\\text{Accuracy} = \\frac{\\text{Number of Correct Predictions}}{\\text{Total Number of Predictions}}\n$$","metadata":{"_uuid":"e8412bc1-a840-47f2-8eb8-4a0d4b552a74","_cell_guid":"831ce24b-b28d-4548-a18e-625a67c33c4c","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_batch_accuracy(output, y):\n    pred = output.argmax(dim=1, keepdim=True)\n    correct = pred.eq(y.view_as(pred)).sum().item()\n    return correct / output.shape[0]","metadata":{"_uuid":"6ec18a63-b0a9-4213-921f-06755f18cc18","_cell_guid":"2883fcfe-fff8-4845-82b6-16092d187bdd","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.000637Z","iopub.execute_input":"2025-03-25T14:09:16.001286Z","iopub.status.idle":"2025-03-25T14:09:16.010974Z","shell.execute_reply.started":"2025-03-25T14:09:16.001247Z","shell.execute_reply":"2025-03-25T14:09:16.010260Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Lists to track metrics","metadata":{"_uuid":"01d49e9a-5fcc-4e74-97b7-fef9b6a8a70f","_cell_guid":"9ad3e663-abb7-4930-96b4-29dedb12cc18","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# We will use these for plotting \ntrain_losses = []\ntrain_accuracies = [] \nvalid_losses = []\nvalid_accuracies = []","metadata":{"_uuid":"1d7e3db7-809f-450e-9cdd-9e6ecc00424d","_cell_guid":"d65dca82-853d-4d08-8f93-762a5d59c825","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.011963Z","iopub.execute_input":"2025-03-25T14:09:16.012662Z","iopub.status.idle":"2025-03-25T14:09:16.022708Z","shell.execute_reply.started":"2025-03-25T14:09:16.012624Z","shell.execute_reply":"2025-03-25T14:09:16.021967Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define training","metadata":{"_uuid":"3de4f04c-1be8-4d06-95f3-32316bf41a75","_cell_guid":"ffb6cd5a-aae5-4039-a977-21333c053e3f","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def train():\n    total_loss = 0\n    total_accuracy = 0\n    \n    model.train()\n    for x, y in train_loader:\n        x, y = x.to(device), y.to(device)\n        output = model(x)\n        optimizer.zero_grad()\n        batch_loss = loss_function(output, y)\n        batch_loss.backward()\n        optimizer.step()\n        \n        total_loss += batch_loss.item()\n        total_accuracy += get_batch_accuracy(output, y)\n    \n    # Calculate average loss and accuracy\n    avg_loss = total_loss / len(train_loader)\n    avg_accuracy = total_accuracy / len(train_loader)\n    \n    # Store metrics\n    train_losses.append(avg_loss)\n    train_accuracies.append(avg_accuracy)\n    \n    print(f'Train - Loss: {avg_loss:.4f} Accuracy: {avg_accuracy:.4f}')\n    return avg_loss, avg_accuracy","metadata":{"_uuid":"362f071f-db5a-4560-9551-a18d22957405","_cell_guid":"d85203cf-c227-40a6-afd3-d92cc96ee66d","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.023794Z","iopub.execute_input":"2025-03-25T14:09:16.024050Z","iopub.status.idle":"2025-03-25T14:09:16.035144Z","shell.execute_reply.started":"2025-03-25T14:09:16.024026Z","shell.execute_reply":"2025-03-25T14:09:16.034356Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Define validation","metadata":{"_uuid":"484a5cd8-21d7-48ff-afb7-8687e41cfa4d","_cell_guid":"1c9dce2e-6418-4fdc-a833-1e6c7d161f0e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def validate():\n    total_loss = 0\n    total_accuracy = 0\n    \n    model.eval()\n    with torch.no_grad():\n        for x, y in valid_loader:\n            x, y = x.to(device), y.to(device)\n            output = model(x)\n            \n            batch_loss = loss_function(output, y)\n            total_loss += batch_loss.item()\n            total_accuracy += get_batch_accuracy(output, y)\n    \n    # Calculate average loss and accuracy        \n    avg_loss = total_loss / len(valid_loader)\n    avg_accuracy = total_accuracy / len(valid_loader)\n    \n    # Store metrics\n    valid_losses.append(avg_loss)\n    valid_accuracies.append(avg_accuracy)\n    \n    print(f'Valid - Loss: {avg_loss:.4f} Accuracy: {avg_accuracy:.4f}')\n    return avg_loss, avg_accuracy","metadata":{"_uuid":"e15c11d0-d100-464f-a664-9ac7a0a6e65f","_cell_guid":"50fe912d-6178-45a4-8c74-deb9fc4a8b0f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.036085Z","iopub.execute_input":"2025-03-25T14:09:16.036401Z","iopub.status.idle":"2025-03-25T14:09:16.050641Z","shell.execute_reply.started":"2025-03-25T14:09:16.036364Z","shell.execute_reply":"2025-03-25T14:09:16.049808Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training config","metadata":{"_uuid":"4d650952-8f13-4351-97fb-4e9f4e5e217e","_cell_guid":"6c392bde-5a88-40f8-bccd-986e6c0372e9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def get_training_config():\n    config = {\n        'learning_rate': 0.001,  # Lower learning rate for Adam\n        'weight_decay': 1e-4,\n        'epochs': 3,\n        'min_lr': 1e-6  # Minimum learning rate for scheduler\n    }\n    return config\n\nconfig = get_training_config()","metadata":{"_uuid":"ff5f14ed-ffab-4aba-9ddb-7ce8167100b3","_cell_guid":"0c7ef00a-c3df-4476-9ed0-1f6b0aa7d511","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.051649Z","iopub.execute_input":"2025-03-25T14:09:16.051899Z","iopub.status.idle":"2025-03-25T14:09:16.061960Z","shell.execute_reply.started":"2025-03-25T14:09:16.051874Z","shell.execute_reply":"2025-03-25T14:09:16.061081Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"optimizer = torch.optim.Adam(\n        model.parameters(),\n        lr=config['learning_rate'],\n        weight_decay=config['weight_decay']\n    )","metadata":{"_uuid":"d09edc18-bc07-4b0c-b408-253a4dca3d8c","_cell_guid":"542a67a8-9412-4964-85d0-667f1a15097f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.062931Z","iopub.execute_input":"2025-03-25T14:09:16.063178Z","iopub.status.idle":"2025-03-25T14:09:16.072491Z","shell.execute_reply.started":"2025-03-25T14:09:16.063153Z","shell.execute_reply":"2025-03-25T14:09:16.071555Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":" # CosineAnnealingLR scheduler\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer, \n        T_max=config['epochs'],\n        eta_min=config['min_lr']\n    )","metadata":{"_uuid":"d768fcee-753f-4740-a250-3c44be36c836","_cell_guid":"b0a2b407-474b-40ed-b864-3c1e89b4bd8f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:09:16.073419Z","iopub.execute_input":"2025-03-25T14:09:16.073701Z","iopub.status.idle":"2025-03-25T14:09:16.083417Z","shell.execute_reply.started":"2025-03-25T14:09:16.073672Z","shell.execute_reply":"2025-03-25T14:09:16.082636Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Send the model to the GPU memory","metadata":{}},{"cell_type":"code","source":"model.to(device)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-25T14:44:50.631084Z","iopub.execute_input":"2025-03-25T14:44:50.631861Z","iopub.status.idle":"2025-03-25T14:44:50.638972Z","shell.execute_reply.started":"2025-03-25T14:44:50.631811Z","shell.execute_reply":"2025-03-25T14:44:50.637943Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training loop with model checkpointing","metadata":{"_uuid":"79646f46-2c62-4ba6-bf86-c66e76dd3c9f","_cell_guid":"e3e9163b-75b7-4a94-919b-44d5a5aec4e6","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# Training loop with model checkpointing\nepochs = config['epochs']\nbest_valid_accuracy = 0\n\n# CosineAnnealingLR scheduler\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n        optimizer, \n        T_max=config['epochs'],\n        eta_min=config['min_lr']\n    )\n\nfor epoch in tqdm(range(epochs), desc='Epochs'):\n    print(f'\\nEpoch: {epoch+1}/{epochs}')\n    \n    train_loss, train_acc = train()\n    valid_loss, valid_acc = validate()\n\n    # Step the scheduler to change the learning rate\n    scheduler.step()\n    \n    # Save model if validation accuracy improves\n    if valid_acc > best_valid_accuracy:\n        best_valid_accuracy = valid_acc\n        torch.save(model.state_dict(), 'best_model.pth')\n        print(f'Model saved! New best validation accuracy: {best_valid_accuracy:.4f}')","metadata":{"_uuid":"d2fd8221-b548-4fac-ba39-47c1f060b980","_cell_guid":"ee4ecdd5-24e8-4144-a610-4b944770209f","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-03-25T14:44:58.477009Z","iopub.execute_input":"2025-03-25T14:44:58.477988Z","iopub.status.idle":"2025-03-25T14:45:55.348518Z","shell.execute_reply.started":"2025-03-25T14:44:58.477948Z","shell.execute_reply":"2025-03-25T14:45:55.347682Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Plot training curves","metadata":{"_uuid":"1b10d1c9-162d-4066-b164-b9cb49b564eb","_cell_guid":"0ab64a58-d170-40bf-a088-dd7b998638a4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# We want to see both validation and training loss going down\nplt.figure(figsize=(10, 5))\n\n# Plot losses\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Training Loss')\nplt.plot(valid_losses, label='Validation Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training vs Validation Loss')\nplt.legend()\n\n# Plot accuracies \nplt.subplot(1, 2, 2)\nplt.plot(train_accuracies, label='Training Accuracy')\nplt.plot(valid_accuracies, label='Validation Accuracy')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Training vs Validation Accuracy')\nplt.legend()\n\nplt.tight_layout()\nplt.show()\n\n# Load best model\nmodel.load_state_dict(torch.load('best_model.pth', weights_only=False))\nprint(f'Loaded best model with validation accuracy: {best_valid_accuracy:.4f}')","metadata":{"_uuid":"3598fb45-12e1-48b4-a063-e428db499e09","_cell_guid":"e4449b8a-a456-46c1-8e7d-5762962b0384","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-03-25T14:45:55.349975Z","iopub.execute_input":"2025-03-25T14:45:55.350256Z","iopub.status.idle":"2025-03-25T14:45:55.875071Z","shell.execute_reply.started":"2025-03-25T14:45:55.350226Z","shell.execute_reply":"2025-03-25T14:45:55.874244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Evaluate on the test data and prepare a submission\nmodel.eval()\npredictions = []\nwith torch.no_grad():\n    for images, _ in test_loader:\n        outputs = model(images.to(device))\n        _, preds = torch.max(outputs, 1)\n        predictions.extend(preds.cpu().numpy())\n\n# Prepare submission\nsubmission = pd.DataFrame({'ImageId': np.arange(1, len(predictions) + 1), 'Label': predictions})\nsubmission.to_csv('submission.csv', index=False)\n\nprint(\"Submission file created: 'submission.csv'\")","metadata":{"_uuid":"8d264f99-114c-4ed5-b323-28399604c1ec","_cell_guid":"a0ca1e39-d51b-4d08-8ff8-f73f9b605a91","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2025-03-25T14:45:55.876309Z","iopub.execute_input":"2025-03-25T14:45:55.876692Z","iopub.status.idle":"2025-03-25T14:46:04.593891Z","shell.execute_reply.started":"2025-03-25T14:45:55.876651Z","shell.execute_reply":"2025-03-25T14:46:04.592997Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Exercises\n\n* Try logging your experiments with Weights and Biases. For this you will need to create an account on https://wandb.ai/. Refer to [this notebook](https://www.kaggle.com/code/andandand/wandb-resnet-applebananaorange) to give you guidance on how to log experiments on wandb through Kaggle.\n* Think about the problem of [data leakage](https://en.wikipedia.org/wiki/Leakage_(machine_learning)), that is having testing data in the training set. How could a Kaggler use it to cheat and achieve a \"perfect\" score of 1.0 accuracy? You can refer to this [Github repo](https://github.com/tesla-is/MNIST-Kaggle-Competition-The-Winning-Solution) and [its discussion on Kaggle](https://www.kaggle.com/c/digit-recognizer/discussion/61480) for ideas.","metadata":{"_uuid":"bcc3116e-1649-4ae7-a046-f875637c0a36","_cell_guid":"9f72386a-7e91-4ac2-9cac-11381dd8a8c9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}}]}