{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Import Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch, torchvision\nimport torch.nn as nn\nfrom torch import optim\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import transforms, models\nfrom PIL import Image\nimport gc\nfrom tqdm import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-13T07:13:23.955888Z","iopub.execute_input":"2022-02-13T07:13:23.956250Z","iopub.status.idle":"2022-02-13T07:13:25.586310Z","shell.execute_reply.started":"2022-02-13T07:13:23.956219Z","shell.execute_reply":"2022-02-13T07:13:25.585562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\ntransform = transforms.Compose([\n        transforms.Resize((224, 224)),\n        transforms.Grayscale(3),\n        transforms.ToTensor(),\n        transforms.Normalize((0.1307,0.1307,0.1307), (0.3081,0.3081,0.3081)),\n    ])\nBATCH_SIZE = 64","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:25.587921Z","iopub.execute_input":"2022-02-13T07:13:25.588203Z","iopub.status.idle":"2022-02-13T07:13:25.636835Z","shell.execute_reply.started":"2022-02-13T07:13:25.588155Z","shell.execute_reply":"2022-02-13T07:13:25.636095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:28.753340Z","iopub.execute_input":"2022-02-13T07:13:28.754004Z","iopub.status.idle":"2022-02-13T07:13:28.760814Z","shell.execute_reply.started":"2022-02-13T07:13:28.753965Z","shell.execute_reply":"2022-02-13T07:13:28.760131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import Data","metadata":{}},{"cell_type":"code","source":"# Open the data in the pandas table\ntrain_df = pd.read_csv('../input/digit-recognizer/train.csv', dtype=np.float32)\ntest_df = pd.read_csv('../input/digit-recognizer/test.csv', dtype=np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:31.747346Z","iopub.execute_input":"2022-02-13T07:13:31.747948Z","iopub.status.idle":"2022-02-13T07:13:38.277238Z","shell.execute_reply.started":"2022-02-13T07:13:31.747909Z","shell.execute_reply":"2022-02-13T07:13:38.276448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:38.278979Z","iopub.execute_input":"2022-02-13T07:13:38.279461Z","iopub.status.idle":"2022-02-13T07:13:38.348011Z","shell.execute_reply.started":"2022-02-13T07:13:38.279423Z","shell.execute_reply":"2022-02-13T07:13:38.347206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:38.351393Z","iopub.execute_input":"2022-02-13T07:13:38.351837Z","iopub.status.idle":"2022-02-13T07:13:38.399986Z","shell.execute_reply.started":"2022-02-13T07:13:38.351800Z","shell.execute_reply":"2022-02-13T07:13:38.399229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# See what the pictures look like\nfig, ax = plt.subplots(2, 5, figsize=(10, 10))  # 2行5列\nfor i in range(2):  # 行\n    for j in range(5):  # 列\n        num = np.random.randint(0, len(train_df) - 1)  # 随机选取图片,这里是Pd的第几行\n        img_np = train_df.iloc[num, 1:].values.reshape(28,28)\n        label = train_df.iloc[num, 0]\n        ax[i, j].imshow(img_np, cmap='gray')\n        ax[i, j].set_title(label)\n        ax[i, j].axis('off')\n    fig.set_figheight(7)\n    fig.set_figwidth(15)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:40.210864Z","iopub.execute_input":"2022-02-13T07:13:40.211621Z","iopub.status.idle":"2022-02-13T07:13:40.785843Z","shell.execute_reply.started":"2022-02-13T07:13:40.211584Z","shell.execute_reply":"2022-02-13T07:13:40.785141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preparing Dataset","metadata":{}},{"cell_type":"code","source":"#  Build CustomDataset\nclass CustomDataset(Dataset):\n    def __init__(self, df, transform, train_data=True):\n        self.df = df\n        self.transform = transform\n        self.train_data = train_data\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        if self.train_data:\n            label = self.df.iloc[index, 0]\n            image_np = self.df.iloc[index, 1:].values.reshape(28,28)\n        else:\n            image_np = self.df.iloc[index].values.reshape(28,28)\n        image_as_Image = Image.fromarray(image_np)\n        if self.transform:\n            image_tensor = self.transform(image_as_Image)\n        if self.train_data:\n            return image_tensor, torch.tensor(label).long() #标签要转换为LongTensor类型\n        return image_tensor","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:45.648083Z","iopub.execute_input":"2022-02-13T07:13:45.648707Z","iopub.status.idle":"2022-02-13T07:13:45.656893Z","shell.execute_reply.started":"2022-02-13T07:13:45.648666Z","shell.execute_reply":"2022-02-13T07:13:45.655829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CustomDataset(train_df, transform, train_data=True)\ntest_dataset = CustomDataset(test_df, transform, train_data=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:46.459144Z","iopub.execute_input":"2022-02-13T07:13:46.459814Z","iopub.status.idle":"2022-02-13T07:13:46.464419Z","shell.execute_reply.started":"2022-02-13T07:13:46.459771Z","shell.execute_reply":"2022-02-13T07:13:46.463449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split train into train and val\ntrain_size = int(0.8 * len(train_dataset))\ntest_size = len(train_dataset) - train_size\ntrain_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [train_size, test_size])","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:48.251705Z","iopub.execute_input":"2022-02-13T07:13:48.251960Z","iopub.status.idle":"2022-02-13T07:13:48.269958Z","shell.execute_reply.started":"2022-02-13T07:13:48.251931Z","shell.execute_reply":"2022-02-13T07:13:48.269290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False )\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:13:49.008715Z","iopub.execute_input":"2022-02-13T07:13:49.009086Z","iopub.status.idle":"2022-02-13T07:13:49.016399Z","shell.execute_reply.started":"2022-02-13T07:13:49.009052Z","shell.execute_reply":"2022-02-13T07:13:49.015455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Build CNN Architecture","metadata":{}},{"cell_type":"code","source":"# Test model on sample example\nnet = models.resnet18(pretrained=True)\nin_features = net.fc.in_features\nnet.fc = nn.Linear(in_features, 10)\nnet = net.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:19:23.610651Z","iopub.execute_input":"2022-02-13T07:19:23.610915Z","iopub.status.idle":"2022-02-13T07:19:23.847838Z","shell.execute_reply.started":"2022-02-13T07:19:23.610885Z","shell.execute_reply":"2022-02-13T07:19:23.847085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.ones(1,3,224,224).to(device)\nnet(x).shape","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:19:25.253486Z","iopub.execute_input":"2022-02-13T07:19:25.254104Z","iopub.status.idle":"2022-02-13T07:19:25.991105Z","shell.execute_reply.started":"2022-02-13T07:19:25.254068Z","shell.execute_reply":"2022-02-13T07:19:25.990409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define loss function and optimize","metadata":{}},{"cell_type":"code","source":"optimizer = optim.Adam(net.parameters(), lr=1e-4)\ncriterion = nn.CrossEntropyLoss(reduction='sum')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:20:13.383775Z","iopub.execute_input":"2022-02-13T07:20:13.384520Z","iopub.status.idle":"2022-02-13T07:20:13.389222Z","shell.execute_reply.started":"2022-02-13T07:20:13.384479Z","shell.execute_reply":"2022-02-13T07:20:13.388245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training and Validation","metadata":{}},{"cell_type":"code","source":"def train_epoch(net, data_loader, device, criterion, optimizer):\n    net.train()\n    per_epoch_sample = len(data_loader.dataset)\n    correct = 0  # 需要两个变量记录正确率和损失\n    losses = 0\n    for batch_idx, (x, y) in enumerate(tqdm(data_loader)):\n        x = x.to(device, non_blocking=True)\n        y = y.to(device, non_blocking=True)\n        output = net(x)\n        loss = criterion(output, y)\n        loss.backward()\n        optimizer.step()\n        optimizer.zero_grad()\n\n        y_predict = torch.max(output, dim=1)[1]\n        correct_count = torch.sum(y == y_predict)\n\n        losses += loss.item()\n        correct += correct_count\n\n    print(correct)\n    loss_per_epoch = losses / per_epoch_sample\n    acc_per_epoch = correct / per_epoch_sample\n\n    del x, y, correct_count, loss, correct\n    gc.collect()\n    torch.cuda.empty_cache()\n\n    return loss_per_epoch, acc_per_epoch","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:20:15.528978Z","iopub.execute_input":"2022-02-13T07:20:15.529254Z","iopub.status.idle":"2022-02-13T07:20:15.539877Z","shell.execute_reply.started":"2022-02-13T07:20:15.529224Z","shell.execute_reply":"2022-02-13T07:20:15.539078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def val_epoch(net, data_loader, device, criterion):\n    net.eval()\n    losses = 0\n    correct = 0\n    per_epoch_sample = len(data_loader.dataset)\n\n    with torch.no_grad():\n        for batch_idx, (x,y) in enumerate(tqdm(data_loader)):\n            x = x.to(device, non_blocking=True)\n            y = y.to(device, non_blocking=True)\n            output = net(x)\n            loss = criterion(output, y)\n\n            y_predict = torch.max(output, dim=1)[1]\n            correct_count = torch.sum(y == y_predict)\n\n            losses += loss.item()\n            correct += correct_count\n\n        acc_per_epoch = correct / per_epoch_sample\n        loss_per_epoch = losses / per_epoch_sample\n\n        del x, y, correct_count, loss, correct\n        gc.collect()\n        torch.cuda.empty_cache()\n\n        return loss_per_epoch, acc_per_epoch","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:20:16.327672Z","iopub.execute_input":"2022-02-13T07:20:16.328220Z","iopub.status.idle":"2022-02-13T07:20:16.335374Z","shell.execute_reply.started":"2022-02-13T07:20:16.328165Z","shell.execute_reply":"2022-02-13T07:20:16.334394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_val_acc = 0\nepochs = 10\ntrain_loss_all, train_acc_all = [], []\nval_loss_all, val_acc_all = [], []\nfor epoch in range(epochs):\n    print('Epoch:{}/{}'.format(epoch+1, epochs))\n    print('-'*10)\n    print('Training')\n    train_loss, train_acc = train_epoch(net, train_loader, device, criterion, optimizer)\n    print('Validing')\n    val_loss, val_acc = val_epoch(net, val_loader, device, criterion)\n\n    print('\\t Train_loss:{:.6f}, Train_acc:{:.6f}'.format(train_loss, train_acc))\n    print('\\t Val Loss: {:.6f}, Val Acc:{:.6f}'.format(val_loss, val_acc))\n    \n    train_loss_all.append(train_loss)\n    train_acc_all.append(train_acc.cpu())\n    val_loss_all.append(val_loss)\n    val_acc_all.append(val_acc.cpu())\n    \n    if val_acc > best_val_acc:\n        best_val_acc = val_loss\n        torch.save(net.state_dict(), 'best_model.pt')","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:26:29.855477Z","iopub.execute_input":"2022-02-13T07:26:29.855958Z","iopub.status.idle":"2022-02-13T07:50:19.378878Z","shell.execute_reply.started":"2022-02-13T07:26:29.855919Z","shell.execute_reply":"2022-02-13T07:50:19.378008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing Model Performance","metadata":{}},{"cell_type":"code","source":"def plot_loss(train_loss, val_loss):\n    plt.figure(figsize=(10, 7))\n    plt.plot(train_loss, color=\"red\", label=\"Trainloss\")\n    plt.plot(val_loss, color=\"orange\", label=\"Testloss\")\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()\ndef plot_acc(train_acc, val_acc):\n    plt.figure(figsize=(10, 7))\n    plt.plot(train_acc, color=\"red\", label=\"Trainloss\")\n    plt.plot(val_acc, color=\"orange\", label=\"Testloss\")\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:50:24.879536Z","iopub.execute_input":"2022-02-13T07:50:24.879797Z","iopub.status.idle":"2022-02-13T07:50:24.889212Z","shell.execute_reply.started":"2022-02-13T07:50:24.879766Z","shell.execute_reply":"2022-02-13T07:50:24.888427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_loss(train_loss_all, val_loss_all)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:50:28.365890Z","iopub.execute_input":"2022-02-13T07:50:28.366149Z","iopub.status.idle":"2022-02-13T07:50:28.577389Z","shell.execute_reply.started":"2022-02-13T07:50:28.366118Z","shell.execute_reply":"2022-02-13T07:50:28.576669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_acc(train_acc_all, val_acc_all)","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:50:38.988716Z","iopub.execute_input":"2022-02-13T07:50:38.988975Z","iopub.status.idle":"2022-02-13T07:50:39.194710Z","shell.execute_reply.started":"2022-02-13T07:50:38.988944Z","shell.execute_reply":"2022-02-13T07:50:39.194034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test Model","metadata":{}},{"cell_type":"code","source":"# Load best model\nnet.load_state_dict(torch.load('best_model.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:50:45.571833Z","iopub.execute_input":"2022-02-13T07:50:45.572110Z","iopub.status.idle":"2022-02-13T07:50:45.617558Z","shell.execute_reply.started":"2022-02-13T07:50:45.572076Z","shell.execute_reply":"2022-02-13T07:50:45.616840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_model():\n    submission = [['ImageId', 'Label']]\n    net.eval()\n\n    image_id = 1\n    with torch.no_grad():\n        for x in tqdm(test_loader):\n            x = x.to(device)\n            output = net(x)\n            pred = torch.max(output, dim=1)[1]\n            for prediction in pred:\n                submission.append([image_id, prediction.item()])\n                image_id += 1\n    return submission","metadata":{"execution":{"iopub.status.busy":"2022-02-13T07:52:14.492345Z","iopub.execute_input":"2022-02-13T07:52:14.492631Z","iopub.status.idle":"2022-02-13T07:52:14.498483Z","shell.execute_reply.started":"2022-02-13T07:52:14.492599Z","shell.execute_reply":"2022-02-13T07:52:14.497772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub = test_model()\nsub_pd = pd.DataFrame(sub)\nsub_pd.columns = sub_pd.iloc[0]\nsub_pd.drop(0, axis=0, inplace=True)\nsub_pd.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-13T07:52:18.031156Z","iopub.execute_input":"2022-02-13T07:52:18.031716Z","iopub.status.idle":"2022-02-13T07:53:25.888754Z","shell.execute_reply.started":"2022-02-13T07:52:18.031673Z","shell.execute_reply":"2022-02-13T07:53:25.886988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}