{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle competition: https://www.kaggle.com/c/digit-recognizer/","metadata":{}},{"cell_type":"markdown","source":"## WANDB link: https://wandb.ai/ales-2000-09/digit_recognizer?workspace=user-ales-2000-09\n## Alessandro Castelli code link: https://www.kaggle.com/code/alessandromajumba/ex3-ml\n\n### Best Score = 0.97875","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-08T17:09:48.843238Z","iopub.execute_input":"2024-01-08T17:09:48.844000Z","iopub.status.idle":"2024-01-08T17:09:48.852620Z","shell.execute_reply.started":"2024-01-08T17:09:48.843960Z","shell.execute_reply":"2024-01-08T17:09:48.851621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to import Torch's libraries","metadata":{}},{"cell_type":"code","source":"import torchvision\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"Label\")\n\nimport wandb\nwandb.login(key=secret_value_0)\nwandb.init(project='digit_recognizer', save_code=True)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:09:48.854551Z","iopub.execute_input":"2024-01-08T17:09:48.854896Z","iopub.status.idle":"2024-01-08T17:10:27.256665Z","shell.execute_reply.started":"2024-01-08T17:09:48.854870Z","shell.execute_reply":"2024-01-08T17:10:27.255680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\n\nA custom dataset which uses the CSV from Kaggle, avoid downloading the dataset from internet","metadata":{}},{"cell_type":"code","source":"class MyMNISTDataset(Dataset):\n    \n    def __init__(self, file_path, transform = transforms.Compose([transforms.ToPILImage()]), test_data=False, use_gpu=torch.cuda.is_available()):\n        # read the data\n        df = pd.read_csv(file_path)\n        # for test data we don't have any target\n        # MNIST images are 28 by 28, grey colors\n        if test_data:\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        self.transform = transform\n        self.use_gpu = use_gpu\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        data = self.transform(self.X[idx])\n        if self.y is not None:\n            target = self.y[idx]\n            if self.use_gpu:\n                data = data.cuda()\n                target = target.cuda()\n            return data, target\n        else:\n            if self.use_gpu:\n                data = data.cuda()\n            return data","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:10:27.257824Z","iopub.execute_input":"2024-01-08T17:10:27.258190Z","iopub.status.idle":"2024-01-08T17:10:27.752677Z","shell.execute_reply.started":"2024-01-08T17:10:27.258137Z","shell.execute_reply":"2024-01-08T17:10:27.751768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data.sampler import SubsetRandomSampler\n\ntransformations=transforms.Compose([transforms.ToPILImage(), \n                                    transforms.ToTensor(), \n                                    transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\ntrain_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/train.csv', transform=transformations, test_data=False)\ntest_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/test.csv', transform=transformations, test_data=True)\n\n# create data loader for train and test set\n\n# Define the size of validation set\nvalidation_split = 0.2\ndataset_size = len(train_dataset)\nindices = list(range(dataset_size))\nsplit = int(np.floor(validation_split * dataset_size))\n\n# Shuffle the indices\nnp.random.shuffle(indices)\n\n# Split the indices into training and validation sets\ntrain_indices, val_indices = indices[split:], indices[:split]\n\n# Create the samplers\ntrain_sampler = SubsetRandomSampler(train_indices)\nval_sampler = SubsetRandomSampler(val_indices)\n\n# Create the data loaders\ntrain_loader = DataLoader(train_dataset, batch_size=64, sampler=train_sampler)\nval_loader = DataLoader(train_dataset, batch_size=64, sampler=val_sampler)\ntest_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)  # For the test set, we use shuffle=False\n","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:10:27.754855Z","iopub.execute_input":"2024-01-08T17:10:27.755223Z","iopub.status.idle":"2024-01-08T17:10:32.504847Z","shell.execute_reply.started":"2024-01-08T17:10:27.755190Z","shell.execute_reply":"2024-01-08T17:10:32.503895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MLP\n\n### Define model architecture\nYou need to reach at least 70% accuracy on the test set","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):    \n    def __init__(self):\n        super(Net, self).__init__()\n        self.fc1 = nn.Linear(28 * 28, 512) # Adjust input size based on your image dimensions\n        self.fc2 = nn.Linear(512, 256)\n        self.fc3 = nn.Linear(256, 10) # Output size corresponds to the number of classes\n        self.dropout = nn.Dropout(0.2)\n        \n    def forward(self, x):\n        x = x.view(-1, 28 * 28)  # Flatten the input\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n\n        x = self.dropout(x)\n        \n        x = self.fc3(x)\n        return x\n\n# Create an instance of the Net model\nmodel = Net()#.to(device)\n\n# If using GPU, move the model to GPU\nif torch.cuda.is_available():\n    model.cuda()\n\n# Print the model architecture\nprint(model)\n","metadata":{"ExecuteTime":{"end_time":"2023-12-08T18:15:01.793286Z","start_time":"2023-12-08T18:15:01.780071Z"},"execution":{"iopub.status.busy":"2024-01-08T17:10:32.506040Z","iopub.execute_input":"2024-01-08T17:10:32.506389Z","iopub.status.idle":"2024-01-08T17:10:32.975486Z","shell.execute_reply.started":"2024-01-08T17:10:32.506356Z","shell.execute_reply":"2024-01-08T17:10:32.974578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init the model and put it on GPU/TPU","metadata":{}},{"cell_type":"code","source":"# let use the model on the GPU\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint('Using device:', device)\n\nprint(model)\n\nlr = 0.1\nwandb.log({'learning rate': lr})\n\n#Cross-Entropy as Loss function\ncriterion = nn.CrossEntropyLoss()\n\n#SGD optimizer\noptimizer = torch.optim.SGD(model.parameters(), lr=lr)\nwandb.log({'optimizer':'SGD'})\n\n#This is another optimizer\n# optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n# wandb.log({'optimizer':'ADAM'})","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:10:32.976755Z","iopub.execute_input":"2024-01-08T17:10:32.977077Z","iopub.status.idle":"2024-01-08T17:10:33.394496Z","shell.execute_reply.started":"2024-01-08T17:10:32.977047Z","shell.execute_reply":"2024-01-08T17:10:33.393694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loop\nLog the accuracy and the loss to wandb","metadata":{}},{"cell_type":"code","source":"# number of epochs \nn_epochs = 40\nmean_train = []\nmean_valid = []\nvalid_acc = []\n#Save on WANDB\nwandb.log({'num_epochs': n_epochs})\n\n#Start training\nfor epoch in range(n_epochs):\n    train_losses = []\n    valid_losses = []\n\n    model.train() \n    for data, target in train_loader:\n        data=data.to(device)\n        target=target.to(device)\n        optimizer.zero_grad()\n        output = model(data)\n        loss = criterion(output, target)\n        loss.backward()\n        optimizer.step()\n        train_losses.append(loss.item())\n        \n#start evaluation\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for data, target in val_loader:\n            data=data.to(device)\n            target=target.to(device)\n            output = model(data)\n            loss = criterion(output, target)\n            valid_losses.append(loss.item())\n            \n            _, predicted = torch.max(output.data, 1)\n            correct += (predicted == target).sum().item()\n            total += target.size(0)\n\n    mean_train.append(np.mean(train_losses))\n    mean_valid.append(np.mean(valid_losses))\n    \n    accuracy = 100*correct/total\n    valid_acc.append(accuracy)\n    #Print the results\n    print('epoch : {}, train loss : {:.4f}, valid loss : {:.4f}, valid acc : {:.2f}%'\\\n         .format(epoch+1, np.mean(train_losses), np.mean(valid_losses), accuracy))\n    \n    #Save on WANDB\n    wandb.log({'train_loss':mean_train[-1]})\n    wandb.log({'val_loss':mean_valid[-1]})\n    wandb.log({'val_accuracy':valid_acc[-1]})","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:10:33.395557Z","iopub.execute_input":"2024-01-08T17:10:33.395819Z","iopub.status.idle":"2024-01-08T17:24:59.727493Z","shell.execute_reply.started":"2024-01-08T17:10:33.395795Z","shell.execute_reply":"2024-01-08T17:24:59.726607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make prediction\nAnd submit to Kaggle for grading","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_loader):\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], columns=['ImageId', 'Label'])\n    out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:24:59.728754Z","iopub.execute_input":"2024-01-08T17:24:59.729099Z","iopub.status.idle":"2024-01-08T17:25:07.438992Z","shell.execute_reply.started":"2024-01-08T17:24:59.729065Z","shell.execute_reply":"2024-01-08T17:25:07.437748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Top 10 misclassified images by class probability","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nbatch_size=64\nunshuffle_train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                           batch_size=batch_size, shuffle=False)\nwith torch.no_grad():\n    model.eval()\n    missclasified = torch.DoubleTensor()\n    for batch_idx, (data, target) in enumerate(unshuffle_train_loader):\n        output = model(data)\n        prob, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        target = target.cpu()\n        prob = prob.cpu().double()\n        missclassified_prob = torch.where(predicted == target, 0., prob)\n        missclasified = torch.cat((missclasified, missclassified_prob), dim=0)\n    most_misclassified = torch.argsort(missclasified, descending=True)\n    top_ten_misclassified = most_misclassified[:10]","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:25:07.442577Z","iopub.execute_input":"2024-01-08T17:25:07.443304Z","iopub.status.idle":"2024-01-08T17:25:20.022530Z","shell.execute_reply.started":"2024-01-08T17:25:07.443267Z","shell.execute_reply":"2024-01-08T17:25:20.021453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for misclassified in top_ten_misclassified:\n    plt.imshow(train_dataset[misclassified][0].cpu().reshape(28,28))\n    with torch.no_grad():\n        data, target = train_dataset[misclassified]\n        data = data.reshape(1, 1, 28,28)\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        plt.title(f'Predicted: {predicted.item()}, Ground truth: {target}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-01-08T17:25:20.023753Z","iopub.execute_input":"2024-01-08T17:25:20.024081Z","iopub.status.idle":"2024-01-08T17:25:22.948181Z","shell.execute_reply.started":"2024-01-08T17:25:20.024050Z","shell.execute_reply":"2024-01-08T17:25:22.947134Z"},"trusted":true},"execution_count":null,"outputs":[]}]}