{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle competition: https://www.kaggle.com/c/digit-recognizer/","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2023-12-21T20:46:40.042573Z","iopub.execute_input":"2023-12-21T20:46:40.042956Z","iopub.status.idle":"2023-12-21T20:46:40.418497Z","shell.execute_reply.started":"2023-12-21T20:46:40.042924Z","shell.execute_reply":"2023-12-21T20:46:40.417544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to import Torch's libraries","metadata":{}},{"cell_type":"code","source":"import torchvision\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:46:40.420428Z","iopub.execute_input":"2023-12-21T20:46:40.421241Z","iopub.status.idle":"2023-12-21T20:46:45.986032Z","shell.execute_reply.started":"2023-12-21T20:46:40.421189Z","shell.execute_reply":"2023-12-21T20:46:45.985045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\n\nA custom dataset which uses the CSV from Kaggle, avoid downloading the dataset from internet","metadata":{}},{"cell_type":"code","source":"class MyMNISTDataset(Dataset):\n    \n    def __init__(self, file_path, transform = transforms.Compose([transforms.ToPILImage()]), test_data=False, use_gpu=torch.cuda.is_available()):\n        # read the data\n        df = pd.read_csv(file_path)\n        # for test data we don't have any target\n        # MNIST images are 28 by 28, grey colors\n        if test_data:\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        self.transform = transform\n        self.use_gpu = use_gpu\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        data = self.transform(self.X[idx])\n        if self.y is not None:\n            target = self.y[idx]\n            if self.use_gpu:\n                data = data.cuda()\n                target = target.cuda()\n            return data, target\n        else:\n            if self.use_gpu:\n                data = data.cuda()\n            return data","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:46:45.987228Z","iopub.execute_input":"2023-12-21T20:46:45.987504Z","iopub.status.idle":"2023-12-21T20:46:46.070715Z","shell.execute_reply.started":"2023-12-21T20:46:45.987479Z","shell.execute_reply":"2023-12-21T20:46:46.069555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\n\ntransformations=transforms.Compose([transforms.ToPILImage(), \n                                    transforms.ToTensor(), \n                                    transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\ntrain_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/train.csv', transform=transformations, test_data=False)\ntest_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/test.csv', transform=transformations, test_data=True)\n# train_dataset, _ = torch.utils.data.random_split(train_dataset, [0.2, 0.8])\n\ntrain_dataset, validation_dataset = torch.utils.data.random_split(train_dataset, [0.8, 0.2])\ntrain_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\ntest_dataloader = DataLoader(test_dataset, batch_size=batch_size)\nvalidation_dataloader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:46:46.073405Z","iopub.execute_input":"2023-12-21T20:46:46.074127Z","iopub.status.idle":"2023-12-21T20:46:51.902879Z","shell.execute_reply.started":"2023-12-21T20:46:46.074095Z","shell.execute_reply":"2023-12-21T20:46:51.901800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MLP\n\n### Define model architecture\nYou need to reach at least 70% accuracy on the test set","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()\n        self.flatten = nn.Flatten()\n        self.linear_relu_stack = nn.Sequential(\n            nn.Linear(28*28, 512),\n            nn.ReLU(),\n            nn.Linear(512, 512),\n            nn.ReLU(),\n            nn.Linear(512, 10),\n        )\n\n    def forward(self, x):\n        x = self.flatten(x)\n        logits = self.linear_relu_stack(x)\n        return logits","metadata":{"ExecuteTime":{"end_time":"2023-12-08T18:15:01.793286Z","start_time":"2023-12-08T18:15:01.780071Z"},"execution":{"iopub.status.busy":"2023-12-21T20:46:51.904560Z","iopub.execute_input":"2023-12-21T20:46:51.905178Z","iopub.status.idle":"2023-12-21T20:46:51.911756Z","shell.execute_reply.started":"2023-12-21T20:46:51.905146Z","shell.execute_reply":"2023-12-21T20:46:51.910750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init the model and put it on GPU/TPU","metadata":{}},{"cell_type":"code","source":"device = (\n    \"cuda\"\n    if torch.cuda.is_available()\n    else \"mps\"\n    if torch.backends.mps.is_available()\n    else \"cpu\"\n)\nprint(f\"Using {device} device\")\nmodel = Net().to(device)","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:46:51.912992Z","iopub.execute_input":"2023-12-21T20:46:51.913328Z","iopub.status.idle":"2023-12-21T20:46:55.118931Z","shell.execute_reply.started":"2023-12-21T20:46:51.913302Z","shell.execute_reply":"2023-12-21T20:46:55.117909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loop\nLog the accuracy and the loss to wandb","metadata":{}},{"cell_type":"code","source":"learning_rate = 5e-3\nepochs = 100\nweight_decay = 96e-5\nmomentum = 0.5\n\n# wandb.init(\n#     project='MNIST_MLP',\n#     save_code=True,\n#     config={\n#         \"learning_rate\": learning_rate,\n#         \"epochs\": epochs,\n#         \"batch_size\": batch_size,\n#         \"weight_decay\": weight_decay,\n#         \"num_training_samples\": len(train_dataset),\n#         \"momentum\": momentum\n#     },\n# )\n\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=weight_decay, momentum=momentum)\n\ndef train_loop(dataloader, model, loss_fn, optimizer):\n    num_batches = len(dataloader)\n    model.train()\n    train_loss = 0\n    for batch, (X, y) in enumerate(dataloader):\n        # Compute prediction and loss\n        pred = model(X)\n        loss = loss_fn(pred, y)\n\n        # Backpropagation\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n        train_loss += loss.item()\n#     wandb.log({\"train_loss\": train_loss / num_batches})\n\n\ndef test_loop(dataloader, model, loss_fn):\n    model.eval()\n    size = len(dataloader.dataset)\n    num_batches = len(dataloader)\n    test_loss, correct = 0, 0\n\n    with torch.no_grad():\n        for X, y in dataloader:\n            pred = model(X)\n            test_loss += loss_fn(pred, y).item()\n            correct += (pred.argmax(1) == y).type(torch.float).sum().item()\n\n    test_loss /= num_batches\n    correct /= size\n#     wandb.log({\"test_loss\": test_loss, \"accuracy\": correct})\n    print(f\"Test Error: \\n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \\n\")\n\nfor t in range(epochs):\n    print(f\"Epoch {t+1}\\n-------------------------------\")\n    train_loop(train_dataloader, model, loss_fn, optimizer)\n#     test_loop(validation_dataloader, model, loss_fn)","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:46:55.120115Z","iopub.execute_input":"2023-12-21T20:46:55.120425Z","iopub.status.idle":"2023-12-21T20:48:21.525451Z","shell.execute_reply.started":"2023-12-21T20:46:55.120397Z","shell.execute_reply":"2023-12-21T20:48:21.523618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make prediction\nAnd submit to Kaggle for grading","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_dataloader):\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], columns=['ImageId', 'Label'])\n    out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:48:21.526942Z","iopub.status.idle":"2023-12-21T20:48:21.527438Z","shell.execute_reply.started":"2023-12-21T20:48:21.527188Z","shell.execute_reply":"2023-12-21T20:48:21.527211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 10 misclassified images by class probability","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nunshuffle_train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                           batch_size=batch_size, shuffle=False)\nwith torch.no_grad():\n    model.eval()\n    missclasified = torch.DoubleTensor()\n    for batch_idx, (data, target) in enumerate(unshuffle_train_loader):\n        output = model(data)\n        prob, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        target = target.cpu()\n        prob = prob.cpu().double()\n        missclassified_prob = torch.where(predicted == target, 0., prob)\n        missclasified = torch.cat((missclasified, missclassified_prob), dim=0)\n    most_misclassified = torch.argsort(missclasified, descending=True)\n    top_ten_misclassified = most_misclassified[:10]","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:48:21.528896Z","iopub.status.idle":"2023-12-21T20:48:21.529354Z","shell.execute_reply.started":"2023-12-21T20:48:21.529113Z","shell.execute_reply":"2023-12-21T20:48:21.529134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for misclassified in top_ten_misclassified:\n    plt.imshow(train_dataset[misclassified][0].cpu().reshape(28,28))\n    with torch.no_grad():\n        data, target = train_dataset[misclassified]\n        data = data.reshape(1, 1, 28,28)\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        plt.title(f'Predicted: {predicted.item()}, Ground truth: {target}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-21T20:48:21.531595Z","iopub.status.idle":"2023-12-21T20:48:21.532092Z","shell.execute_reply.started":"2023-12-21T20:48:21.531843Z","shell.execute_reply":"2023-12-21T20:48:21.531867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}