{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\n\nimport torchvision.datasets as dsets\nimport torchvision.transforms as transforms\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-12T16:05:05.101316Z","iopub.execute_input":"2023-02-12T16:05:05.101696Z","iopub.status.idle":"2023-02-12T16:05:05.113870Z","shell.execute_reply.started":"2023-02-12T16:05:05.101666Z","shell.execute_reply":"2023-02-12T16:05:05.112797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_size = 28\nnum_classes = 10\nnum_epochs = 20\nbatch_size = 64","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:05.116980Z","iopub.execute_input":"2023-02-12T16:05:05.117810Z","iopub.status.idle":"2023-02-12T16:05:05.124573Z","shell.execute_reply.started":"2023-02-12T16:05:05.117769Z","shell.execute_reply":"2023-02-12T16:05:05.123545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nsubmission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:05.141478Z","iopub.execute_input":"2023-02-12T16:05:05.142013Z","iopub.status.idle":"2023-02-12T16:05:08.853923Z","shell.execute_reply.started":"2023-02-12T16:05:05.141976Z","shell.execute_reply":"2023-02-12T16:05:08.852966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dsetsMNIST(torch.utils.data.Dataset):\n    def __init__(self, data, transform=None, labeled=True):\n        self.data = data\n        self.transform = transform\n        self.labeled = labeled\n        \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, index):\n        item = self.data.iloc[index]\n        if self.labeled:\n            x = item[1:].values.astype(np.uint8).reshape((28,28)) #pixel\n            y = item[0] #label\n        else:\n            x = item[0:].values.astype(np.uint8).reshape((28,28))\n            y = 0\n            \n        if self.transform is not None:\n            x = self.transform(x)\n            \n        return x, y","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:08.855978Z","iopub.execute_input":"2023-02-12T16:05:08.856597Z","iopub.status.idle":"2023-02-12T16:05:08.864688Z","shell.execute_reply.started":"2023-02-12T16:05:08.856557Z","shell.execute_reply":"2023-02-12T16:05:08.863713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomAffine(degrees=10, translate=(0, 0.1), scale=(0.9, 1.1)),\n    transforms.ToTensor(),\n    transforms.Normalize([0.1307], [0.3081])\n])\n\ntransform_valid = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.1307], [0.3081])\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.1307], [0.3081])\n])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:08.867924Z","iopub.execute_input":"2023-02-12T16:05:08.868723Z","iopub.status.idle":"2023-02-12T16:05:08.876263Z","shell.execute_reply.started":"2023-02-12T16:05:08.868685Z","shell.execute_reply":"2023-02-12T16:05:08.875327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#split train data into validation data and train data\n#validation data is used to detect overfitting and adjust parameters\ntrain_dataset = dsetsMNIST(train, transform=transform_train)\nvalidation_dataset = dsetsMNIST(train, transform=transform_valid)\ntest_dataset = dsetsMNIST(test, transform=transform_test, labeled=False)\n\nindices = range(len(train_dataset))\nindices_val = indices[:5000]\nindices_train = indices[5000:]\n\nsampler_val = torch.utils.data.sampler.SubsetRandomSampler(indices_val)\nsampler_train = torch.utils.data.sampler.SubsetRandomSampler(indices_train)\n\ntrain_loader = torch.utils.data.DataLoader(dataset=train_dataset,\n                                            batch_size=batch_size,\n                                            sampler=sampler_train) \nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_dataset,\n                                                batch_size=batch_size,\n                                                sampler=sampler_val) \ntest_loader = torch.utils.data.DataLoader(dataset=test_dataset,\n                                          batch_size=batch_size) ","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:08.878941Z","iopub.execute_input":"2023-02-12T16:05:08.880510Z","iopub.status.idle":"2023-02-12T16:05:08.889282Z","shell.execute_reply.started":"2023-02-12T16:05:08.880455Z","shell.execute_reply":"2023-02-12T16:05:08.888291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 1000\n\nmuteimg = train_dataset[idx][0].numpy()\nplt.imshow(muteimg[0,...])\nprint('tag: ',train_dataset[idx][1])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:08.892047Z","iopub.execute_input":"2023-02-12T16:05:08.892842Z","iopub.status.idle":"2023-02-12T16:05:09.076832Z","shell.execute_reply.started":"2023-02-12T16:05:08.892814Z","shell.execute_reply":"2023-02-12T16:05:09.075934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"depth = [4, 8]\n# depth of two convolution layers (num of feature maps)\nclass ConvNet(nn.Module):\n    # invoked when a Convnet object is created (net=ConvNet())\n    def __init__(self):\n        super(ConvNet, self).__init__()\n        \n        self.conv1 = nn.Conv2d(1, 4, 5, padding=2) # convolution layer (input channel, output channel, window, padding)\n        self.pool = nn.MaxPool2d(2, 2) # pooling layer window size 2*2\n        self.conv2 = nn.Conv2d(depth[0], depth[1], 5, padding=2) # second conv layer \n        \n        self.fc1 = nn.Linear(image_size // 4 * image_size // 4 * depth[1], 512) # input size is the tiling of last layer's cube, output layer is 512 nodes\n        self.fc2 = nn.Linear(512, num_classes) # output is the num of classes required\n        \n    def forward(self, x):\n        # x size (batch_size, image_channels, image_width, image_height)\n        x = F.relu(self.conv1(x)) # conv layer, relu is activation func which could prevent overfitting\n        # x size (batch_size, num_filters, image_width, image_height)\n        x = self.pool(x) # pooling, reduce image size\n        # x size (batch_size, depth[0], image_width/2, image_height/2)\n        x = F.relu(self.conv2(x)) # conv layer window size = 5, input=depth[0]=4, output=depth[1]=8\n        # x size (batch_size, depth[1], image_width/2, image_height/2)\n        x = self.pool(x) # pooling\n        # x size (batch_size, depth[1], image_width/4, image_height/4)\n        \n        x = x.view(-1, image_size // 4 * image_size // 4 * depth[1])\n        # x size (batch_size, depth[1]*image_width/4*image_height/4)\n        \n        x = F.relu(self.fc1(x))\n        # x size (batch_size, 512)\n        x = F.dropout(x, training=self.training) #dropout neurons with the probability of 0.5\n        x = self.fc2(x)\n        # x size (batch_size, num_classes)\n        \n        x = F.log_softmax(x, dim=0) #output layer is log_softmax log(p(x))\n        return x\n    \n    def retrieve_features(self, x):\n        # return feature maps of convolution layers\n        feature_map1 = F.relu(self.conv1(x)) #conv\n        x = self.pool(feature_map1) #pooling\n        feature_map2 = F.relu(self.conv2(x))\n        return(feature_map1, feature_map2)","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:09.078309Z","iopub.execute_input":"2023-02-12T16:05:09.078871Z","iopub.status.idle":"2023-02-12T16:05:09.090213Z","shell.execute_reply.started":"2023-02-12T16:05:09.078833Z","shell.execute_reply":"2023-02-12T16:05:09.089063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rightness(predictions, labels):\n    pred = torch.max(predictions.data, 1)[1]\n    rights = pred.eq(labels.data.view_as(pred)).sum()\n    pred = pred.tolist()\n    return rights, len(labels), pred","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:09.091707Z","iopub.execute_input":"2023-02-12T16:05:09.092320Z","iopub.status.idle":"2023-02-12T16:05:09.102573Z","shell.execute_reply.started":"2023-02-12T16:05:09.092280Z","shell.execute_reply":"2023-02-12T16:05:09.101616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = ConvNet()\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n\nrecord = []\nweights = []\n\nfor epoch in range(num_epochs):\n    train_rights = []\n    \n    for batch_idx, (data, target) in enumerate(train_loader):\n        data, target = data.clone().requires_grad_(True), target.clone().detach()\n        \n        net.train()\n        \n        output = net(data)\n        loss = criterion(output, target) #compare output and target\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        right = rightness(output, target)\n        train_rights.append(right)\n        \n        if batch_idx % 100 == 0:\n            \n            net.eval()\n            val_rights = []\n            \n            for (data, target) in validation_loader:\n                data, target = data.clone().requires_grad_(True), target.clone().detach()\n                output = net(data)\n                right = rightness(output, target)\n                val_rights.append(right)\n                \n            # train[0]/train[1] is the classification accuracy of training set\n            # val[0]/val[1] is the classification accuracy of validation set\n            train_r = (sum([tup[0] for tup in train_rights]), sum([tup[1] for tup in train_rights]))\n            val_r = (sum([tup[0] for tup in val_rights]), sum([tup[1] for tup in val_rights]))\n            \n            print('epoch:{} [{}/{} ({:.0f}%)]\\t, Loss: {:.6f}\\t, training accuracy:{:.2f}%\\t, validating accuracy:{:.2f}%'.format(\n                epoch, batch_idx * len(data), len(train_loader.dataset),\n                100. * batch_idx / len(train_loader), loss.data,\n                100. * train_r[0] / train_r[1],\n                100. * val_r[0] / val_r[1]))\n            \n            record.append((100 - 100. * train_r[0] / train_r[1], 100 - 100. * val_r[0] / val_r[1]))\n            \n            weights.append([net.conv1.weight.data.clone(), net.conv1.bias.data.clone(),\n                           net.conv2.weight.data.clone(),net.conv2.bias.data.clone()])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:05:09.104041Z","iopub.execute_input":"2023-02-12T16:05:09.104425Z","iopub.status.idle":"2023-02-12T16:19:17.931003Z","shell.execute_reply.started":"2023-02-12T16:05:09.104384Z","shell.execute_reply":"2023-02-12T16:19:17.930047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (10, 7))\nplt.plot(record)\nplt.xlabel('Steps')\nplt.ylabel('Error rate')","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:19:17.932360Z","iopub.execute_input":"2023-02-12T16:19:17.932709Z","iopub.status.idle":"2023-02-12T16:19:18.131090Z","shell.execute_reply.started":"2023-02-12T16:19:17.932674Z","shell.execute_reply":"2023-02-12T16:19:18.130210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.eval()\nvals = []\n\nfor data, target in test_loader:\n    data, target = data.clone().detach().requires_grad_(True), target.clone().detach()\n    output = net(data)\n    val = rightness(output, target)\n    vals.append(val)\n\nrights = (sum([tup[0] for tup in vals]), sum([tup[1] for tup in vals]))\nrights_rate = 1.0 * rights[0] / rights[1]\n\nresult = []\nfor tup in vals:\n    if result == []:\n        result = tup[2]\n    else:\n        result += tup[2]\n\nresult = torch.IntTensor(result)\nresult","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:19:18.134040Z","iopub.execute_input":"2023-02-12T16:19:18.134758Z","iopub.status.idle":"2023-02-12T16:19:28.536771Z","shell.execute_reply.started":"2023-02-12T16:19:18.134729Z","shell.execute_reply":"2023-02-12T16:19:28.535691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = 27999\nmuteimg = test_dataset[idx][0].numpy()\nplt.imshow(muteimg[0,...])","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:19:28.538394Z","iopub.execute_input":"2023-02-12T16:19:28.538777Z","iopub.status.idle":"2023-02-12T16:19:28.719343Z","shell.execute_reply.started":"2023-02-12T16:19:28.538742Z","shell.execute_reply":"2023-02-12T16:19:28.718290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(np.c_[np.arange(1, len(test_dataset)+1)[:, None], result.numpy()],\n                         columns=['ImageId', 'Label'])\nsubmission.to_csv('submission.csv',index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-02-12T16:19:28.720571Z","iopub.execute_input":"2023-02-12T16:19:28.720917Z","iopub.status.idle":"2023-02-12T16:19:28.760134Z","shell.execute_reply.started":"2023-02-12T16:19:28.720869Z","shell.execute_reply":"2023-02-12T16:19:28.759099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}