{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install timm","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install torchinfo==1.6.3","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install sklearn","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nfrom torchvision import transforms, models\nimport pandas as pd\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom PIL import Image\nfrom tqdm import tqdm\nimport os\nfrom sklearn.metrics import accuracy_score\nimport timm\nfrom tqdm import tqdm  \nfrom torchinfo import summary\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom ranger import Ranger","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Write necessary utils","metadata":{}},{"cell_type":"code","source":"root_in = '/root/autodl-tmp/Dogs' #Folder with input (image, lable)\nroot_out = '/root/autodl-tmp/Dogs/Output' #Folder with output image \nhave_index = False # If the breed label have been map to a index","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_pre_access(file, output):\n    '''transfer train label into index'''\n    labels = pd.read_csv(file, index_col='id')\n    labels_map = dict()\n    labels['label_index'] = torch.zeros((labels.shape[0])).type(torch.int32).numpy()\n    for i, label in enumerate(labels.breed.unique()):\n        labels_map[i] = label\n        labels.loc[labels.breed == label, 'label_index'] = i\n    labels.to_csv(output)\n    \n    return labels_map","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if have_index:\n    labels_map = {}\n    t_data = pd.read_csv(os.path.join(root_out,'labels_index.csv'))\n    \n    def label_f(m):\n        labels_map[int(m.label_index)] = m.label\n        \n    t_data.apply(label_f,axis=1)\nelse:\n    labels_map = data_pre_access(os.path.join(root_in,'labels.csv'), output=os.path.join(root_out,'labels_index.csv'))","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(labels_map)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_data_pre_access(direction, output):\n    '''list the test image direction in a csv'''\n    folder = os.listdir(direction)\n    files = [file for file in folder if os.path.isfile(os.path.join(direction, file))]\n    \n    test_dataframe = pd.DataFrame({'id':files})\n    \n    def split_jpg(m):\n        id = str(m.id).split('.jpg')\n        s.append(id[0])\n    s = []\n    test_dataframe.apply(split_jpg, axis=1)\n    test_dataframe['id'] = s\n    \n    test_dataframe.to_csv(os.path.join(output, 'test.csv'))\n    return test_dataframe","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = test_data_pre_access(os.path.join(root_in,'test'),root_out)\ntest_data","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''test_people = test_data_pre_access(os.path.join('/root/autodl-tmp/people'),root_out)\ntest_people'''","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.set_index('id')","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dogs_Train_Dataset(Dataset):\n    '''Train Dataset'''\n    def __init__(self, file_in, transform=None):\n        self.img_paths = pd.read_csv(file_in)\n        self.transform = transform\n        \n    def __len__(self):\n        return self.img_paths.shape[0]\n    \n    def __getitem__(self, index):\n        img = Image.open(os.path.join(root_in, 'train', self.img_paths.iloc[index, 0] + '.jpg'))\n        label_index = self.img_paths.iloc[index, 2]\n        if self.transform:\n            img = self.transform(img)\n        return img, label_index\n    ","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dogs_Test_Dataset(Dogs_Train_Dataset):\n    '''Test Dataset'''\n    def __getitem__(self, index):\n        img = Image.open(os.path.join(root_in, 'test', self.img_paths.iloc[index, 1] + '.jpg'))\n        if self.transform:\n            img = self.transform(img)\n        return img","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Accumulator():\n    '''A counter util, which count the float value of the input'''\n    def __init__(self, nums):\n        self.metric = list(torch.zeros((nums,)).numpy())\n        \n    def __getitem__(self, index):\n        return self.metric[index]\n    \n    def add(self, *args):\n        for i, item in enumerate(args):\n            self.metric[i] += float(item)\n    ","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a = Accumulator(2)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#a.add(0,0)\nmetric = a \nmetric[0]","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy(y_hat, y):\n    '''used to count the right type'''\n    y_hat = y_hat.exp().argmax(dim=1)\n    y_hat.reshape((-1))\n    y.reshape((-1))\n    return accuracy_score(y.cpu().numpy(), y_hat.cpu().numpy(), normalize=False)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_accuracy(net, data_iter, device=None):\n    '''Evalue the valid dataset'''\n    if isinstance(net, nn.Module):\n        net.eval()\n        if not device:\n            device = next(iter(net.parameters())).device\n    \n    metric = Accumulator(2)\n    with torch.no_grad():\n        for X, y in data_iter:\n            if isinstance(X, list):\n                X = [x.to(device) for x in X]\n            else:\n                X = X.to(device)\n            y.to(device)\n            metric.add(accuracy(net(X), y), y.numel())\n    return metric[0] / metric[1]","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_test(net, test_iter, device=None):\n    '''Inference'''\n    if isinstance(net, nn.Module):\n        net.eval()\n        if not device:\n            device = next(iter(net.parameters())).device\n    y = []\n    net.to(device)\n    softmax = nn.Softmax(dim=1)\n    with torch.no_grad():\n        for X in test_iter:\n            if isinstance(X, list):\n                X = [x.to(device) for x in X]\n            else:\n                X = X.to(device)\n            y += softmax(net(X).cpu())\n\n    return list(Y.numpy() for Y in y)\n    ","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data","metadata":{}},{"cell_type":"code","source":"train_transform = transforms.Compose([\n    transforms.Resize(256),\n    # 从图像中心裁切224x224大小的图片\n    transforms.CenterCrop(224),\n    # 随机裁剪图像，所得图像为原始面积的0.2到1之间，高宽比在3/4和4/3之间。\n    # 然后，缩放图像以创建224 x 224的新图像\n    transforms.RandomResizedCrop(224, scale=(0.2, 1.0), ratio=(3.0 / 4.0, 4.0 / 3.0)),\n    transforms.RandomHorizontalFlip(),\n    # 随机更改亮度，对比度和饱和度\n    #transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4),\n    transforms.ToTensor(),\n    # 标准化图像的每个通道\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])\n\nval_test_transform = transforms.Compose([\n    transforms.Resize(256),\n    # 从图像中心裁切224x224大小的图片\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize([0.485, 0.456, 0.406],[0.229, 0.224, 0.225])])","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dog_train_dataset = Dogs_Train_Dataset(os.path.join(root_out, 'labels_index.csv'),transform=train_transform)\ndog_val_dataset = Dogs_Train_Dataset(os.path.join(root_out, 'labels_index.csv'),transform=val_test_transform)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_length = int(len(dog_train_dataset) * 0.90) # train_length is not used\nvalid_length = len(dog_train_dataset) - train_length # define the valid_dataset's length\n_, dog_val_dataset = random_split(dog_val_dataset, [train_length, valid_length])\n\n#dog_val_dataset.transform = val_test_transform\ndog_test_dataset = Dogs_Test_Dataset(os.path.join(root_out, 'test.csv'),transform=val_test_transform)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dog_train_loader = DataLoader(dog_train_dataset, batch_size=32, shuffle=True, drop_last=True, num_workers=20)\ndog_val_loader = DataLoader(dog_val_dataset, batch_size=32, shuffle=True, num_workers=20)\ndog_test_loader = DataLoader(dog_test_dataset, batch_size=16, shuffle=False, num_workers=20)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finetune Pretrained Model","metadata":{}},{"cell_type":"code","source":"model = timm.create_model('resnest200e', pretrained=True, num_classes=len(labels_map))\nsummary(model,input_size = (32, 3, 224, 224))","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_name = [name for name,_ in model.named_parameters()] # All parameters name\nlayer_name = [name for name,_ in model.named_modules()] # All layers name","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_name[-12:], layer_name[-20:]\nlen(param_name)","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model = nn.Sequential(model, nn.ReLU(), nn.Dropout(0.3), nn.Linear(len(labels_map), len(labels_map)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load model state from the .params ","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load('/root/autodl-tmp/Dogs/Output/Dogs_Breed.params'))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze_pretrained_layers(model):\n    '''Freeze all layers except the last layer(fc or classifier)'''\n    for param in model.parameters():\n            param.requires_grad = False\n    #nn.init.xavier_normal_(model.fc.weight)\n    #nn.init.zeros_(model.fc.bias)\n    model.fc.weight.requires_grad = True\n    model.fc.bias.requires_grad = True","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"freeze_pretrained_layers(model)\nmodel.fc.weight.requires_grad","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def debarcle_layers(model, num_debarcle):\n    '''Debarcle From the last [-1]layer to the [-num_debarcle] layers, \n    approximately(for there is Conv2d which has only weight parameter)'''\n    num_debarcle *= 2\n    param_debarcle = param_name[-num_debarcle:]\n    if param_debarcle[0].split('.')[-1] == 'bias':\n        param_debarcle = param_name[-(num_debarcle + 1):]\n    for name, param in model.named_parameters():\n        param.requires_grad = True if name in param_debarcle else False","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"debarcle_layers(model, 200) # Debarcle the last 200 layers()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(net, num_epochs, train_loader, val_loader, loss, lr, device, lr_min=1e-5, weight_decay=0, optim='sgd', use_amp=False, init=True, scheduler_type='Cosine'):\n    '''Parameters:\n        lr(float): the begining learning rate\n        lr_min(float): min learning rate\n        optim(String): the optimizer type\n        use_amp(Boolean): Use mixed precision on GPU or not\n        init(Boolean): Need initial the layers parameter or not\n        scheduler_type(String): Learning rate scheduler\n        \n       Detail:\n        The train process will save the model's parameter every 10 epochs.\n        Every epoch, scheduler update once, and evaluate the train_dataset's accuracy and print it to std 5 times, \n        print the valid_dataset's accuracy once.\n        If the valid accuracy >= 0.90, save the model's parameters as well.\n    '''\n    def init_xavier(m):\n        if type(m) == nn.Linear or type(m) == nn.Conv2d:\n            nn.init.xavier_normal_(m.weight)\n    \n    if init:\n        net.apply(init_xavier)\n    \n    print('training on', device)\n    net.to(device)\n    \n    if optim == 'sgd':\n        optimizer = torch.optim.SGD((param for param in net.parameters() if param.requires_grad), lr=lr, weight_decay=weight_decay)\n    elif optim == 'adam':\n        optimizer = torch.optim.Adam((param for param in net.parameters() if param.requires_grad), lr=lr, weight_decay=weight_decay)\n    elif optim =='adamW':\n        optimizer = torch.optim.AdamW((param for param in net.parameters() if param.requires_grad), lr=lr, weight_decay=weight_decay)\n    elif optim == 'ranger':\n        optimizer = Ranger((param for param in net.parameters() if param.requires_grad), lr=lr, weight_decay=weight_decay)\n        \n    Value_train_l = list()\n    Value_train_acc = list()\n    Value_test_acc = list()\n    \n    scaler = torch.cuda.amp.GradScaler(enabled=use_amp) # mixed_precison\n    \n    if scheduler_type == 'Cosine':\n        scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=lr_min)\n    \n    num_batches = len(train_loader)\n    for epoch in range(num_epochs):\n        net.train()\n        metric = Accumulator(3)\n        \n        for i, (X, y) in enumerate(tqdm(train_loader)):\n            \n            X ,y = X.to(device), y.to(device)\n            \n            with torch.cuda.amp.autocast(enabled=use_amp):\n                y_hat = net(X)\n                l = loss(y_hat, y)\n                \n            scaler.scale(l).backward()\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            \n            with torch.no_grad():\n                metric.add(l * X.shape[0], accuracy(y_hat, y), X.shape[0])\n                \n            train_l = metric[0] / metric[2]\n            train_acc = metric[1] / metric[2]\n            if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:\n                print(epoch + (i + 1) / num_batches,\n                             'train_l train_acc\\t',(train_l, train_acc,None))\n                \n                Value_train_l.append(train_l)\n                Value_train_acc.append(train_acc)\n                Value_test_acc.append(None)\n            \n        scheduler.step()\n\n        test_acc = evaluate_accuracy(net, val_loader, device=device)\n        print('lr = ', optimizer.param_groups[0]['lr'])\n        print(epoch + 1,'test_acc\\t', (None, None, test_acc))\n\n        Value_train_l.append(None)\n        Value_train_acc.append(None)\n        Value_test_acc.append(test_acc)\n        if epoch % 10 == 0 or test_acc >= 0.90:\n            torch.save(net.state_dict(),os.path.join(root_out, 'Dogs_Breed_' + str(epoch + 1) + '.params'))\n    \n        record_data = pd.DataFrame(zip(Value_train_l, Value_train_acc, Value_test_acc))    \n        record_data.to_csv(os.path.join(root_out, 'Record_Dogs_Breed_ResNest.csv')) \n        \n    torch.save(net.state_dict(),os.path.join(root_out, 'Dogs_Breed.params'))\n    print(f'loss {train_l:.3f}, train acc {train_acc:.3f}, '\n          f'test acc {test_acc:.3f}')\n    print(f'on {str(device)}')","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''Maps the model's breed order into submission breed order'''\ns = ''\nwith open('/root/autodl-tmp/Dogs/Output/breed.txt') as f:\n    while(True):\n        c = f.read()\n        if c == '':\n            break\n        else:\n            s += c\nls = s.split('\\n')\nls = ls[0:len(ls)-1]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#people_dataset = Dogs_Test_Dataset(os.path.join(root_out, 'test.csv'),transform=val_test_transform)\n#people_loader = DataLoader(people_dataset, shuffle=False)","metadata":{"tags":[],"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_map.get(0)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result_index","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr, epochs, device = 3e-4, 30, 'cuda' if torch.cuda.is_available() else 'cpu'\nlr_min = 1e-5\n#train_model(model, epochs, dog_train_loader, dog_val_loader, nn.CrossEntropyLoss(), lr, device, lr_min=lr_min, optim='ranger', use_amp=True, init=False, scheduler_type='Cosine')\n\nresult_index = predict_test(model, dog_test_loader, device=device)\nresult = pd.read_csv(os.path.join(root_out, 'test.csv'))\nfor i in range(len(labels_map)):\n    result[labels_map.get(i)] = [x[i] for x in result_index]\n#result_breed = [labels_map.get(i) for i in result_index]\n#result['breed'] = result_breed\na = pd.DataFrame(result)\nresult = result.set_index('id')[ls]\nresult.to_csv(os.path.join(root_out, 'submission.csv'))\n","metadata":{"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.memory_snapshot()","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.memory_stats()","metadata":{"tags":[],"collapsed":true,"jupyter":{"outputs_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result.to_csv(os.path.join(root_out, 'submission.csv'))","metadata":{"tags":[],"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result","metadata":{"jupyter":{"source_hidden":true,"outputs_hidden":true},"tags":[],"collapsed":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"a.set_index('id')","metadata":{"jupyter":{"source_hidden":true,"outputs_hidden":true},"tags":[],"collapsed":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"s = [labels_map.get(i) for i in a]\ns","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}