{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle competition: https://www.kaggle.com/c/digit-recognizer/","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n# Best Score: 0.97835\n# Kaggle: https://www.kaggle.com/code/aakash143rathee/exercise-3/notebook\n# Wandb: https://wandb.ai/aakashrathee/MNIST_MLP/workspace?workspace=user-aakashrathee\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-07T14:54:16.311781Z","iopub.execute_input":"2024-01-07T14:54:16.312140Z","iopub.status.idle":"2024-01-07T14:54:16.678666Z","shell.execute_reply.started":"2024-01-07T14:54:16.312109Z","shell.execute_reply":"2024-01-07T14:54:16.677801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to import Torch's libraries","metadata":{}},{"cell_type":"code","source":"import torchvision\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset, random_split\nimport matplotlib.pyplot as plt\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:54:19.436566Z","iopub.execute_input":"2024-01-07T14:54:19.437391Z","iopub.status.idle":"2024-01-07T14:54:23.214952Z","shell.execute_reply.started":"2024-01-07T14:54:19.437356Z","shell.execute_reply":"2024-01-07T14:54:23.214033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BATCH = 256\nLEARING_RATE = 1e-3\nEPOCHS = 100\nLOSS_FN = 'NLLLoss'\nOPTIM = 'SGD'\n\n\"\"\"\nwandb.init(project='MNIST_MLP', save_code=True, config={\n    'batch size': BATCH,\n    'learning rate': LEARING_RATE,\n    'epochs': EPOCHS,\n    'loss function': LOSS_FN,\n    'OPTIMIZER': OPTIM\n})\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:54:33.834469Z","iopub.execute_input":"2024-01-07T14:54:33.835333Z","iopub.status.idle":"2024-01-07T14:55:12.091917Z","shell.execute_reply.started":"2024-01-07T14:54:33.835300Z","shell.execute_reply":"2024-01-07T14:55:12.090996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\n\nA custom dataset which uses the CSV from Kaggle, avoid downloading the dataset from internet","metadata":{}},{"cell_type":"code","source":"class MyMNISTDataset(Dataset):\n    \n    def __init__(self, file_path, transform = transforms.Compose([transforms.ToPILImage()]), test_data=False, use_gpu=torch.cuda.is_available()):\n        # read the data\n        df = pd.read_csv(file_path)\n        # for test data we don't have any target\n        # MNIST images are 28 by 28, grey colors\n        if test_data:\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        self.transform = transform\n        self.use_gpu = use_gpu\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        data = self.transform(self.X[idx])\n        if self.y is not None:\n            target = self.y[idx]\n            if self.use_gpu:\n                data = data.cuda()\n                target = target.cuda()\n            return data, target\n        else:\n            if self.use_gpu:\n                data = data.cuda()\n            return data","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:55:12.093501Z","iopub.execute_input":"2024-01-07T14:55:12.093860Z","iopub.status.idle":"2024-01-07T14:55:12.864143Z","shell.execute_reply.started":"2024-01-07T14:55:12.093833Z","shell.execute_reply":"2024-01-07T14:55:12.863283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformations=transforms.Compose([transforms.ToPILImage(), \n                                    transforms.ToTensor(), \n                                    transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\ntrain_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/train.csv', transform=transformations, test_data=False)\ntest_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/test.csv', transform=transformations, test_data=True)\n\ntrain_labels = [x.item() for x in list(train_dataset.y)]\nfor i in range(10):\n    print(f'{i} -> {train_labels.count(i)} ({100 * train_labels.count(i) / len(train_dataset):.3f}%)')\n\ntrain_dataset, val_dataset = random_split(train_dataset, [0.9, 0.1])\n# TODO split train between validation and training set\n# create data loader for train and test set\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=BATCH, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:55:12.865378Z","iopub.execute_input":"2024-01-07T14:55:12.865739Z","iopub.status.idle":"2024-01-07T14:55:19.446496Z","shell.execute_reply.started":"2024-01-07T14:55:12.865707Z","shell.execute_reply":"2024-01-07T14:55:19.445658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = []\nfor _, y in train_loader:\n    train_labels += [x.item() for x in y]\n\nval_labels = []\nfor _, y in val_loader:\n    val_labels += [x.item() for x in y]\n\nprint(f'Train: {len(train_labels)}')\nfor i in range(10):\n    cnt = train_labels.count(i)\n    print(f'{i} -> {cnt} ({100 * cnt / len(train_labels):.3f}%)')\n    \nprint(f'Validation: {len(val_labels)}')\nfor i in range(10):\n    cnt = val_labels.count(i)\n    print(f'{i} -> {cnt} ({100 * cnt / len(val_labels):.3f}%)')","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:55:19.449246Z","iopub.execute_input":"2024-01-07T14:55:19.449627Z","iopub.status.idle":"2024-01-07T14:55:32.908433Z","shell.execute_reply.started":"2024-01-07T14:55:19.449565Z","shell.execute_reply":"2024-01-07T14:55:32.907459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MLP\n\n### Define model architecture\nYou need to reach at least 70% accuracy on the test set","metadata":{}},{"cell_type":"code","source":"# TODO define a multilayer perceptron for image recognition\n\ndef _weights_init(m):\n    torch.nn.init.xavier_normal_(m.weight)\n    torch.nn.init.constant_(m.bias, 0)\n    return m\n\nclass Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()        \n        self.model = nn.Sequential(\n            _weights_init(nn.Linear(28*28, 512)),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            _weights_init(nn.Linear(512, 512)),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            _weights_init(nn.Linear(512, 10)),\n            nn.LogSoftmax(dim=1)\n        )\n    def forward(self, x):\n        out = x.view(-1, 28 * 28)\n        return self.model(out)","metadata":{"ExecuteTime":{"end_time":"2023-12-08T18:15:01.793286Z","start_time":"2023-12-08T18:15:01.780071Z"},"execution":{"iopub.status.busy":"2024-01-07T14:55:32.909715Z","iopub.execute_input":"2024-01-07T14:55:32.910087Z","iopub.status.idle":"2024-01-07T14:55:33.363113Z","shell.execute_reply.started":"2024-01-07T14:55:32.910060Z","shell.execute_reply":"2024-01-07T14:55:33.362340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init the model and put it on GPU/TPU","metadata":{}},{"cell_type":"code","source":"# TODO\nmodel = Net()\nmodel = model.to('cuda')\n\nloss_fn = nn.NLLLoss()\noptimizer = optim.SGD(model.parameters(), lr=LEARING_RATE, momentum=0.8)\nschedular = optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=5)\n\ntrain_loss = []\nval_loss = []\ntrain_accuracy = []\nval_accuracy = []\nlrs = []","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:55:33.364236Z","iopub.execute_input":"2024-01-07T14:55:33.364516Z","iopub.status.idle":"2024-01-07T14:55:33.841954Z","shell.execute_reply.started":"2024-01-07T14:55:33.364491Z","shell.execute_reply":"2024-01-07T14:55:33.840973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loop\nLog the accuracy and the loss to wandb","metadata":{}},{"cell_type":"code","source":"min_val_loss = 1e10\n\nfor e in range(EPOCHS): \n    \n    if (e+1) % 10 == 0:\n        print(f'{e+1}/{EPOCHS}')\n        \n    lrs.append(schedular.optimizer.param_groups[0]['lr'])\n    \n    model.train()\n    running_loss = 0\n    running_correct = 0\n    for image, label in train_loader:\n        optimizer.zero_grad()\n        pred = model(image)\n        \n        loss = loss_fn(pred, label)\n        loss.backward()\n        optimizer.step()\n        \n        running_correct += (pred.argmax(dim=1) == label).float().sum().item()\n        running_loss += loss.item()\n    \n    train_loss.append(running_loss / len(train_dataset))\n    train_accuracy.append(100 * running_correct / len(train_dataset))\n    \"\"\"\n    wandb.log({\n        'epoch': e,\n        'training_loss': running_loss / len(train_dataset), \n        'training_accuracy': 100 * running_correct / len(train_dataset)\n    }, commit=False)\n    \"\"\"\n    \n    model.eval()\n    running_loss = 0\n    running_correct = 0\n    with torch.no_grad():\n        for image, label in val_loader:\n            pred = model(image)\n            loss = loss_fn(pred, label)\n            \n            running_correct += (pred.argmax(dim=1) == label).float().sum().item()\n            running_loss += loss.item()\n            \n    val_loss.append(running_loss / len(val_dataset))\n    val_accuracy.append(100 * running_correct / len(val_dataset))\n    \n    schedular.step(running_loss / len(val_dataset))\n    \"\"\"\n    wandb.log({\n        'validation_loss': running_loss / len(val_dataset), \n        'validation_accuracy': 100 * running_correct / len(val_dataset)\n    })\n    \"\"\"\n    \n    if running_loss < min_val_loss:\n        print(f'saving model: {e+1}')\n        torch.save(model.state_dict(), 'model.pt')\n        min_val_loss = running_loss\n\nprint(f'Last Train Accuracy: {train_accuracy[-1]}')\nprint(f'Last Validation Accuracy: {val_accuracy[-1]}')","metadata":{"execution":{"iopub.status.busy":"2024-01-07T14:55:33.843384Z","iopub.execute_input":"2024-01-07T14:55:33.843789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 1, figsize=(10, 10))\n\nax[0].plot(list(range(len(train_loss))), train_loss, list(range(len(val_loss))), val_loss)\nax[0].legend([\"Train\", \"Validation\"])\nax[0].set_title(\"Loss\")\n\nax[1].plot(list(range(len(train_accuracy))), train_accuracy, list(range(len(val_accuracy))), val_accuracy)\nax[1].set_title(\"Accuracy\")\nax[1].axis(ymin=0,ymax=120)\n\nax[2].plot(list(range(len(lrs))), lrs)\nax[2].set_title(\"Learning Rate\")\nax[1].axis(ymin=0)\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make prediction\nAnd submit to Kaggle for grading","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load('model.pt'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_loader = DataLoader(test_dataset, batch_size=1)\n\nwith torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_loader):\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], columns=['ImageId', 'Label'])\n    out_df.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 10 misclassified images by class probability","metadata":{}},{"cell_type":"code","source":"unshuffle_train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                           batch_size=BATCH, shuffle=False)\nwith torch.no_grad():\n    model.eval()\n    missclasified = torch.DoubleTensor()\n    for batch_idx, (data, target) in enumerate(unshuffle_train_loader):\n        output = model(data)\n        prob, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        target = target.cpu()\n        prob = prob.cpu().double()\n        missclassified_prob = torch.where(predicted == target, 0., prob)\n        missclasified = torch.cat((missclasified, missclassified_prob), dim=0)\n    most_misclassified = torch.argsort(missclasified, descending=True)\n    top_ten_misclassified = most_misclassified[:10]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for misclassified in top_ten_misclassified:\n    plt.imshow(train_dataset[misclassified][0].cpu().reshape(28,28))\n    with torch.no_grad():\n        data, target = train_dataset[misclassified]\n        data = data.reshape(1, 1, 28,28)\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        plt.title(f'Predicted: {predicted.item()}, Ground truth: {target}')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}