{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os \nimport torch \nimport torchvision\nfrom torchvision import models\nfrom torch import nn\nimport math\ndata_dir = \"/kaggle/input/dog-breed-identification\"\nimport collections\n\ndef read_csv_labels(fname):\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\ndef tv_split(labels, valr):\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    n_valid_per_label = max(1, math.floor(n * valr))\n    valid_labels, train_labels = {},{}\n    count_v = {};count_t = {}\n    for name, label in labels.items():\n        if label not in count_v or count_v.get(label, 0 ) < n_valid_per_label :\n            count_v[label] = count_v.get(label, 0 ) + 1\n            valid_labels[name] = label\n        else :\n            count_t[label] = count_t.get(label, 0 ) + 1\n            train_labels[name] = label\n    return train_labels, valid_labels\n    \ndef ds_process(data_dir, valr):\n    labels_dir = read_csv_labels(os.path.join(data_dir,'labels.csv'))\n    train_labels, valid_labels = tv_split(labels_dir, valr)\n    return train_labels, valid_labels\n\ntransform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomResizedCrop(\n        224, scale = (0.08, 1.0), ratio = (3.0/4.0, 4.0/3.0)\n    ),\n    torchvision.transforms.ColorJitter(\n        brightness=0.4,contrast=0.4,saturation=0.4\n    ),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.485,0.465,0.406],[0.229,0.224,0.225])\n])\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-25T16:30:19.132182Z","iopub.execute_input":"2024-10-25T16:30:19.132539Z","iopub.status.idle":"2024-10-25T16:30:23.542028Z","shell.execute_reply.started":"2024-10-25T16:30:19.132502Z","shell.execute_reply":"2024-10-25T16:30:23.541255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import imageio","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:23.543552Z","iopub.execute_input":"2024-10-25T16:30:23.543951Z","iopub.status.idle":"2024-10-25T16:30:23.595754Z","shell.execute_reply.started":"2024-10-25T16:30:23.543917Z","shell.execute_reply":"2024-10-25T16:30:23.595066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomImageDataset(torch.utils.data.Dataset):\n    def __init__(self, image_paths:list(), labels:list(), str_int:dict(), transform=None):\n        \"\"\"\n        Args:\n            image_paths (list): 图像文件路径的列表。\n            labels (list): 每个图像对应的标签列表。\n            transform (callable, optional): 可选的转换操作。\n        \"\"\"\n#         print(type(image_paths), type(labels), type(str_int), type(transform))\n        self.image_paths = image_paths\n        self.labels = labels\n        self.str_int = str_int\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, idx):\n        # 读取图像\n        image = imageio.imread(self.image_paths[idx])\n\n        # 如果图像是灰度图，将其转换为RGB\n        if len(image.shape) == 2:\n            image = np.stack([image] * 3, axis=-1)\n        elif image.shape[2] == 4:\n            # 如果图像有Alpha通道，移除它\n            image = image[:, :, :3]\n\n        # 将 NumPy 数组转换为 PIL Image\n        image = torchvision.transforms.ToPILImage()(image)\n\n        # 应用转换\n        if self.transform:\n            image = self.transform(image)\n\n        # 获取标签\n        \n        label = self.labels[idx]\n        zeros = torch.zeros(120)\n        zeros[self.str_int[label]] = 1.0\n        return image, zeros\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:23.596827Z","iopub.execute_input":"2024-10-25T16:30:23.597128Z","iopub.status.idle":"2024-10-25T16:30:23.605944Z","shell.execute_reply.started":"2024-10-25T16:30:23.597096Z","shell.execute_reply":"2024-10-25T16:30:23.604799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_name_labels, valid_name_labels = ds_process(\"/kaggle/input/dog-breed-identification\",0.2)\ntrain_files, valid_files = [],[]\ntrain_labels, valid_labels = [],[]\n# print(train_name_labels.keys())\n\nfor dirname, _, filenames in os.walk(\"/kaggle/input/dog-breed-identification/train\"):\n    for filename in filenames:\n        name = filename.split('.')[0]\n        if (name in train_name_labels):\n            train_files.append(os.path.join(dirname, filename))\n            train_labels.append(train_name_labels[name])\n        elif (name in valid_name_labels):\n            valid_files.append(os.path.join(dirname, filename))\n            valid_labels.append(valid_name_labels[name])\n            \n\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:23.608668Z","iopub.execute_input":"2024-10-25T16:30:23.609042Z","iopub.status.idle":"2024-10-25T16:30:28.761069Z","shell.execute_reply.started":"2024-10-25T16:30:23.609Z","shell.execute_reply":"2024-10-25T16:30:28.760277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_str_int_dir = {}\nfor i in train_labels:\n    if i not in label_str_int_dir:\n        label_str_int_dir[i] = len(label_str_int_dir)\ntrain_dataset = CustomImageDataset(train_files, train_labels,label_str_int_dir,transform_train)\nvalid_dataset = CustomImageDataset(valid_files, valid_labels,label_str_int_dir, transform_train)\ntrain_iter = torch.utils.data.DataLoader(train_dataset, 10, shuffle=False)\nvalid_iter = torch.utils.data.DataLoader(valid_dataset, 10,shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.762223Z","iopub.execute_input":"2024-10-25T16:30:28.762533Z","iopub.status.idle":"2024-10-25T16:30:28.770208Z","shell.execute_reply.started":"2024-10-25T16:30:28.762494Z","shell.execute_reply":"2024-10-25T16:30:28.76924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class YZCNet(nn.Module):\n    def __init__(self, num_classes=120):\n        super(YZCNet, self).__init__()\n        # 加载预训练的ResNet50模型\n        self.resnet50 = models.resnet50(pretrained=True)\n        # 替换最后的avgpool和fc层\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.flatten = nn.Flatten()\n        self.fc = nn.Linear(2048, num_classes)  # 2048是ResNet50输出的特征维度\n\n    def forward(self, x):\n        # 使用预训练的ResNet50特征提取部分\n        x = self.resnet50.conv1(x)\n        x = self.resnet50.bn1(x)\n        x = self.resnet50.relu(x)\n        x = self.resnet50.maxpool(x)\n\n        x = self.resnet50.layer1(x)\n        x = self.resnet50.layer2(x)\n        x = self.resnet50.layer3(x)\n        x = self.resnet50.layer4(x)\n\n        # 自定义的avgpool, flatten和全连接层\n        x = self.avgpool(x)\n        x = self.flatten(x)\n        x = self.fc(x)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.771451Z","iopub.execute_input":"2024-10-25T16:30:28.772171Z","iopub.status.idle":"2024-10-25T16:30:28.794197Z","shell.execute_reply.started":"2024-10-25T16:30:28.772137Z","shell.execute_reply":"2024-10-25T16:30:28.793412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class YZCNetSmall(nn.Module):\n    def __init__(self, num_classes=120):\n        super(YZCNetSmall, self).__init__()\n        # 加载预训练的ResNet18模型\n        self.resnet18 = models.resnet18(pretrained=True)\n        # 替换最后的avgpool和fc层\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.flatten = nn.Flatten()\n        self.fc = nn.Linear(512, num_classes)  # ResNet18输出的特征维度是512\n\n    def forward(self, x):\n        # 使用预训练的ResNet18特征提取部分\n        x = self.resnet18.conv1(x)\n        x = self.resnet18.bn1(x)\n        x = self.resnet18.relu(x)\n        x = self.resnet18.maxpool(x)\n\n        x = self.resnet18.layer1(x)\n        x = self.resnet18.layer2(x)\n        x = self.resnet18.layer3(x)\n        x = self.resnet18.layer4(x)\n\n        # 自定义的avgpool, flatten和全连接层\n        x = self.avgpool(x)\n        x = self.flatten(x)\n        x = self.fc(x)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.795207Z","iopub.execute_input":"2024-10-25T16:30:28.795486Z","iopub.status.idle":"2024-10-25T16:30:28.807366Z","shell.execute_reply.started":"2024-10-25T16:30:28.795456Z","shell.execute_reply":"2024-10-25T16:30:28.806469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.808362Z","iopub.execute_input":"2024-10-25T16:30:28.808653Z","iopub.status.idle":"2024-10-25T16:30:28.816881Z","shell.execute_reply.started":"2024-10-25T16:30:28.808622Z","shell.execute_reply":"2024-10-25T16:30:28.815954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluate_loss(data_iter, net, devices):\n    loss = nn.CrossEntropyLoss(reduction = 'none')\n    l_sum, n = 0.0, 0\n    for features, labels in data_iter:\n        features, labels = features.to(devices[0]), labels.to(devices[0])\n        output = net(features)\n        l = loss(output, labels)\n        l_sum += l.sum()\n        n += labels.numel()\n    return (l_sum/n).to('cpu')\ndef do_train(net, train_iter, valid_iter, num_epochs, lr, wd, devices, lr_period, lr_decay):\n    net = net.to(devices[0])\n    trainer = torch.optim.SGD(net.parameters(), lr = lr, momentum=0.9,weight_decay=wd)\n    num_batches = len(train_iter)\n    loss = nn.CrossEntropyLoss(reduction = 'none')\n    \n    for epoch in range(1, num_epochs + 1):\n        \n        for i , (features, labels) in enumerate(train_iter):\n            features, labels = features.to(devices[0]), labels.to(devices[0])\n            trainer.zero_grad()\n            output = net(features)\n            exit(0)\n            l = loss(output, labels).sum()\n            l.backward()\n            trainer.step()\n        if (valid_iter) is not None:\n            valid_loss = evaluate_loss(valid_iter, net, devices)\n            print(valid_loss)\n            torch.save(net.state_dict(), f'model_weights{epoch}.pth')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.818022Z","iopub.execute_input":"2024-10-25T16:30:28.818339Z","iopub.status.idle":"2024-10-25T16:30:28.829866Z","shell.execute_reply.started":"2024-10-25T16:30:28.818308Z","shell.execute_reply":"2024-10-25T16:30:28.828807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = YZCNetSmall()\ndo_train(net, train_iter, valid_iter, 10, 1e-4, 1e-4, [torch.device('cuda'),1], 2, 0.9)","metadata":{"execution":{"iopub.status.busy":"2024-10-25T16:30:28.832099Z","iopub.execute_input":"2024-10-25T16:30:28.832398Z","iopub.status.idle":"2024-10-25T16:32:33.358679Z","shell.execute_reply.started":"2024-10-25T16:30:28.832367Z","shell.execute_reply":"2024-10-25T16:32:33.357318Z"},"trusted":true},"execution_count":null,"outputs":[]}]}