{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Kaggle competition: https://www.kaggle.com/c/digit-recognizer/","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-02-04T17:19:57.182302Z","iopub.execute_input":"2022-02-04T17:19:57.182583Z","iopub.status.idle":"2022-02-04T17:19:57.211883Z","shell.execute_reply.started":"2022-02-04T17:19:57.182505Z","shell.execute_reply":"2022-02-04T17:19:57.211069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We need to import Torch's libraries","metadata":{}},{"cell_type":"code","source":"import torchvision\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\nimport wandb\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wand_key\")\n\nwandb.login(key=secret_value_0)\nwandb.init(project='MNIST_MLP', save_code=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:19:57.213404Z","iopub.execute_input":"2022-02-04T17:19:57.213722Z","iopub.status.idle":"2022-02-04T17:20:08.068588Z","shell.execute_reply.started":"2022-02-04T17:19:57.213684Z","shell.execute_reply":"2022-02-04T17:20:08.067754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data preparation\n\nA custom dataset which uses the CSV from Kaggle, avoid downloading the dataset from internet","metadata":{}},{"cell_type":"code","source":"class MyMNISTDataset(Dataset):\n    \n    def __init__(self, file_path, transform = transforms.Compose([transforms.ToPILImage()]), test_data=False, use_gpu=torch.cuda.is_available()):\n        # read the data\n        df = pd.read_csv(file_path)\n        # for test data we don't have any target\n        # MNIST images are 28 by 28, grey colors\n        if test_data:\n            self.X = df.values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = None\n        else:\n            self.X = df.iloc[:,1:].values.reshape((-1,28,28)).astype(np.uint8)[:,:,:,None]\n            self.y = torch.from_numpy(df.iloc[:,0].values)\n        self.transform = transform\n        self.use_gpu = use_gpu\n    \n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        data = self.transform(self.X[idx])\n        if self.y is not None:\n            target = self.y[idx]\n            if self.use_gpu:\n                data = data.cuda()\n                target = target.cuda()\n            return data, target\n        else:\n            if self.use_gpu:\n                data = data.cuda()\n            return data","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:08.072462Z","iopub.execute_input":"2022-02-04T17:20:08.072682Z","iopub.status.idle":"2022-02-04T17:20:08.652066Z","shell.execute_reply.started":"2022-02-04T17:20:08.072655Z","shell.execute_reply":"2022-02-04T17:20:08.651319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transformations=transforms.Compose([transforms.ToPILImage(), \n                                    transforms.ToTensor(), \n                                    transforms.Normalize(mean=(0.5,), std=(0.5,))])\n\ntrain_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/train.csv', transform=transformations, test_data=False)\ntest_dataset = MyMNISTDataset('/kaggle/input/digit-recognizer/test.csv', transform=transformations, test_data=True)\n\n# TODO split train between validation and training set\ntraining_size = int(0.7 * len(train_dataset))\nvalidation_size = len(train_dataset) - training_size\ntrain, val = torch.utils.data.random_split(train_dataset, [training_size, validation_size], generator=torch.Generator().manual_seed(0))\n\n# create data loader for train and test set\ndata_loader_train = torch.utils.data.DataLoader(train, batch_size=32, shuffle=True)\ndata_loader_val = torch.utils.data.DataLoader(val, batch_size=32, shuffle=True)\ntest_loader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:08.653436Z","iopub.execute_input":"2022-02-04T17:20:08.653682Z","iopub.status.idle":"2022-02-04T17:20:15.293044Z","shell.execute_reply.started":"2022-02-04T17:20:08.653649Z","shell.execute_reply":"2022-02-04T17:20:15.292273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## MLP\n\n### Define model architecture\nYou need to reach at least 70% accuracy on the test set","metadata":{}},{"cell_type":"code","source":"class Net(nn.Module):\n    def __init__(self):\n        super(Net, self).__init__()\n        self.conv1 = nn.Sequential(         \n            nn.Conv2d(\n                in_channels=1,              \n                out_channels=16,            \n                kernel_size=5,              \n                stride=1,                   \n                padding=2,                  \n            ),                              \n            nn.ReLU(),                      \n            nn.MaxPool2d(kernel_size=2),    \n        )\n        self.conv2 = nn.Sequential(         \n            nn.Conv2d(16, 32, 5, 1, 2),     \n            nn.ReLU(),                      \n            nn.MaxPool2d(2),                \n        )\n        # fully connected layer, output 10 classes\n        self.out = nn.Linear(32 * 7 * 7, 10)\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.conv2(x)\n        # flatten the output of conv2 to (batch_size, 32 * 7 * 7)\n        x = x.view(x.size(0), -1)       \n        output = self.out(x)\n        return output    \n","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:15.294983Z","iopub.execute_input":"2022-02-04T17:20:15.295453Z","iopub.status.idle":"2022-02-04T17:20:15.860480Z","shell.execute_reply.started":"2022-02-04T17:20:15.295414Z","shell.execute_reply":"2022-02-04T17:20:15.859770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO\n\n#class Net(nn.Module):    \n #   def __init__(self, input_size=784, output_size=10, layers=[120,84]):\n  #      super(Net, self).__init__()\n   #     #TODO\n    #    self.d1 = nn.Linear(input_size,layers[0]) #layer1\n     #   self.d2 = nn.Linear(layers[0],layers[1]) #layer2\n      #  self.d3 = nn.Linear(layers[1],output_size) #output layer\n\n #   def forward(self, x):\n  #     x = F.relu(self.d1(x))\n   #     x = F.relu(self.d2(x))\n    #    x = self.d3(x)\n     #   return F.log_softmax(x, dim=1)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:15.861740Z","iopub.execute_input":"2022-02-04T17:20:15.861998Z","iopub.status.idle":"2022-02-04T17:20:16.389551Z","shell.execute_reply.started":"2022-02-04T17:20:15.861956Z","shell.execute_reply":"2022-02-04T17:20:16.388818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"28*28","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:16.390822Z","iopub.execute_input":"2022-02-04T17:20:16.391089Z","iopub.status.idle":"2022-02-04T17:20:17.200642Z","shell.execute_reply.started":"2022-02-04T17:20:16.391056Z","shell.execute_reply":"2022-02-04T17:20:17.199956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Init the model and put it on GPU","metadata":{}},{"cell_type":"code","source":"# TODO\nmodel = Net()\nmodel.cuda()\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:17.201852Z","iopub.execute_input":"2022-02-04T17:20:17.202138Z","iopub.status.idle":"2022-02-04T17:20:20.528099Z","shell.execute_reply.started":"2022-02-04T17:20:17.202102Z","shell.execute_reply":"2022-02-04T17:20:20.527349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loop\nLog the accuracy and the loss to wandb","metadata":{}},{"cell_type":"code","source":"# TODO\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\nnum_epochs = 100\ntrain_losses = []\nval_losses = []\ntrain_correct = []\nval_correct = []\n\nfor epoch in range(num_epochs):\n    trn_corr = 0\n    val_corr = 0\n    #TODO TRAINING LOOP\n    for i, y in data_loader_train:\n        output = model(i) # Forward pass \n        loss = criterion(output, y)\n        \n        trn_predicted = torch.argmax(output.data,1)[1].data.squeeze()\n        batch_corr = (trn_predicted == y).sum().item() / float(y.size(0))\n        trn_corr += batch_corr\n        \n        optimizer.zero_grad() # please don't forget!\n        loss.backward() # remember: You need to tell wrt to what the gradient is computed\n        optimizer.step() # do a step in the gradient direction\n    \n    train_losses.append(loss)\n    train_correct.append(trn_corr)\n    \n    with torch.no_grad():\n        #TODO VALIDATION LOOP\n        for z, j in data_loader_val:\n            y_val = model(z)\n            loss2 = criterion(y_val,j)\n            \n            val_predicted = torch.argmax(y_val.data,1)[1].data.squeeze()\n            val_corr += (val_predicted == j).sum().item() / float(j.size(0))\n    \n    val_losses.append(loss2)\n    val_correct.append(val_corr)\n    \n    print(f'epoch {epoch + 1}, training_loss {loss.item()}, validation_loss {loss2.item()}, Train acccuracy {train_correct[-1]}, Validation acccuracy {val_correct[-1]}')\n    wandb.log({'num_epochs': num_epochs, 'training_loss': loss,'Train acccuracy': train_correct, 'validation_loss': loss2, 'Validation acccuracy': val_correct})\n\n    # SAVE THE MODEL\n    torch.save(model.state_dict(), 'my_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:20:20.529224Z","iopub.execute_input":"2022-02-04T17:20:20.529634Z","iopub.status.idle":"2022-02-04T17:41:01.427880Z","shell.execute_reply.started":"2022-02-04T17:20:20.529598Z","shell.execute_reply":"2022-02-04T17:41:01.426547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Make prediction\nAnd submit to Kaggle for grading","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_loader):\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:,None], test_pred.numpy()], columns=['ImageId', 'Label'])\n    out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:41:01.430524Z","iopub.execute_input":"2022-02-04T17:41:01.432367Z","iopub.status.idle":"2022-02-04T17:41:08.268184Z","shell.execute_reply.started":"2022-02-04T17:41:01.432328Z","shell.execute_reply":"2022-02-04T17:41:08.267487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 10 most misclafficied image","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\nunshuffle_train_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                           batch_size=32, shuffle=False)\nwith torch.no_grad():\n    model.eval()\n    missclasified = torch.DoubleTensor()\n    for batch_idx, (data, target) in enumerate(unshuffle_train_loader):\n        output = model(data)\n        prob, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n        target = target.cpu()\n        prob = prob.cpu().double()\n        missclassified_prob = torch.where(predicted == target, 0., prob)\n        missclasified = torch.cat((missclasified, missclassified_prob), dim=0)\n    most_misclassified = torch.argsort(missclasified, descending=True)\n    top_ten_misclassified = most_misclassified[:10]","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:41:08.271325Z","iopub.execute_input":"2022-02-04T17:41:08.271794Z","iopub.status.idle":"2022-02-04T17:41:19.210156Z","shell.execute_reply.started":"2022-02-04T17:41:08.271758Z","shell.execute_reply":"2022-02-04T17:41:19.209434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for misclassified in top_ten_misclassified:\n    plt.imshow(train_dataset[misclassified][0].cpu().reshape(28,28))\n    with torch.no_grad():\n        data, target = train_dataset[misclassified]\n        data = data.reshape(1, 1, 28,28)\n        output = model(data)\n        _, predicted = torch.max(output.data, 1)\n        plt.title(f'Predicted: {predicted.item()}, Ground truth: {target}')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-04T17:41:19.512747Z","iopub.execute_input":"2022-02-04T17:41:19.514408Z","iopub.status.idle":"2022-02-04T17:41:22.686124Z","shell.execute_reply.started":"2022-02-04T17:41:19.514370Z","shell.execute_reply":"2022-02-04T17:41:22.685207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}