{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport skimage.io as skio\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nfrom torchvision.models.resnet import resnet34,resnet18\nfrom torch.utils.data import DataLoader,random_split\nimport torch\nimport cv2\nimport torchvision.transforms\nfrom torch.utils.data import Dataset\nfrom PIL import Image","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:35:56.376401Z","iopub.execute_input":"2023-08-10T12:35:56.377461Z","iopub.status.idle":"2023-08-10T12:35:56.384593Z","shell.execute_reply.started":"2023-08-10T12:35:56.377412Z","shell.execute_reply":"2023-08-10T12:35:56.383826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CashDataset(Dataset):\n    def __init__(self, data_dir, transform):\n        \"\"\"\n        纸币分类任务的Dataset\n        :param data_dir: str, 数据集所在路径\n        :param transform: torch.transform，数据预处理\n        \"\"\"\n        # self.label_name = {\"Health\": 0, \"Other\": 1,\"Rust\":2}\n        self.data_info = self.get_img_info(data_dir)  # data_info存储所有图片路径和标签，在DataLoader中通过index读取样本\n        self.transform = transform\n    def __getitem__(self, index):\n        path_img, label = self.data_info[index]\n        img=skio.imread(path_img)\n        \n        img = img.transpose(2, 0, 1).astype('float')[:125] #class 'numpy.ndarray'>\n        #img = Image.open(path_img).convert('RGB')\n        img = torch.from_numpy(img)\n        img=img.float()\n        # if self.transform is not None:\n        #     img = self.transform(img)\n        return img, label\n    def __len__(self):\n        return len(self.data_info)\n    @staticmethod\n    def get_img_info(data_dir):\n        data_info = list()\n        label_name = {\"Health\": 0, \"Other\": 1, \"Rust\": 2}\n        for root, dirs, files in os.walk(data_dir):\n            # 遍历类别\n            for sub_dir in dirs:\n                img_names = os.listdir(os.path.join(root, sub_dir))\n                img_names = list(filter(lambda x: x.endswith('.tif'), img_names))\n                # 遍历图片\n                for i in range(len(img_names)):\n                    img_name = img_names[i]\n                    path_img = os.path.join(root, sub_dir, img_name)\n                    label = label_name[sub_dir]\n                    data_info.append((path_img, int(label)))\n        return data_info","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:36:01.620193Z","iopub.execute_input":"2023-08-10T12:36:01.620869Z","iopub.status.idle":"2023-08-10T12:36:01.633114Z","shell.execute_reply.started":"2023-08-10T12:36:01.620827Z","shell.execute_reply":"2023-08-10T12:36:01.631810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.transforms as transforms\nbath_size=64\ntransforms=torchvision.transforms.Compose([transforms.Resize(64),transforms.ToTensor()])  \ntrain_data=CashDataset(data_dir=\"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-P1/train/\",\n                       transform=transforms)\ntrain_size=int(len(train_data)*0.8)\nval_size=len(train_data)-train_size\ntrain_dataset,val_dataset=random_split(train_data,[train_size,val_size])\nprint(len(train_dataset))\n\ntrain_loader=DataLoader(train_dataset,bath_size,shuffle=True)\nval_loader=DataLoader(val_dataset,bath_size,shuffle=True)\n# net=torchvision.models.resnet18(pretrained=True)\ndef plot_loss(x, history):\n    plt.plot(x, history['val_loss'], label='val', marker='o')\n    plt.plot(x, history['train_loss'], label='train', marker='o')\n    plt.title('Loss per epoch')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(), plt.grid()\n    #plt.savefig('./save/loss.png')\n    plt.show()\ndef plot_acc(x, history):\n    plt.plot(x, history['train_acc'], label='train_acc', marker='x')\n    plt.plot(x, history['val_acc'], label='val_acc', marker='x')\n    plt.title('Acc per epoch')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(), plt.grid()\n    #plt.savefig('./save/acc.png')\n    plt.show()\n\nclass Dr(nn.Module):\n    def __init__(self):\n        super(Dr,self).__init__()\n        self.modle=nn.Sequential(\n            #xse_resnet18(c_in=125,n_out=3),\n            nn.Conv2d(125, 250, 3, 1, 1),\n            nn.MaxPool2d(2,2,0),\n            nn.ReLU(),\n            nn.Conv2d(250, 100, 3, 1, 17),\n            nn.MaxPool2d(2,2,0),\n            nn.ReLU(),\n            nn.Conv2d(100, 3, 3, 1, 17),\n            nn.MaxPool2d(2,2,0),\n            nn.ReLU(),\n            resnet18(),\n            nn.ReLU(),\n            nn.Linear(1000, 3)\n        )\n    def forward(self,x):\n        x=self.modle(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:39:31.348743Z","iopub.execute_input":"2023-08-10T12:39:31.349169Z","iopub.status.idle":"2023-08-10T12:39:31.547080Z","shell.execute_reply.started":"2023-08-10T12:39:31.349136Z","shell.execute_reply":"2023-08-10T12:39:31.545968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if os.path.exists(\"/kaggle/working//bestone1.pth\"):\n    pass\nelse:\n    dr=Dr()\n    loss_fn=nn.CrossEntropyLoss()\n    learndata=0.0001\n    optimsizer=torch.optim.SGD(dr.parameters(),lr=learndata)\n    total_train_step = 0\n    #epoch=20\n    epoch=30\n    best_acc=0\n    t_loss = []\n    v_loss = []\n    t_acc = []\n    v_acc = []\n    for i in range(epoch):\n        print(\"-------第 {} 轮训练开始-------\".format(i + 1))\n        run_loss=0\n        train_acc=0\n        for data in train_loader:\n            img, target = data\n            optimsizer.zero_grad()\n    \n            output = dr(img)\n            output_tensor=torch.max(output,dim=1)[1]\n            acc=(output_tensor==target).sum().item()\n            if total_train_step==1:\n                print(f\"这是预测值{output}\")\n                print(f\"这是真实值{target}\")\n                print(f\"这是预测目标值{output_tensor}\")\n\n            loss = loss_fn(output, target)\n\n            loss.backward()\n            optimsizer.step()\n            total_train_step = total_train_step + 1\n            run_loss+=loss.item()\n            train_acc+=acc\n\n        print(f\"这是训练集{i+1}的总损失{run_loss/train_size}\")\n        print(f\"这是训练集{i+ 1}的正确率{train_acc/train_size}\")\n        val_losssum=0\n        val_accsum=0\n        with torch.no_grad():\n            for data in val_loader:\n                valimg,valtarget=data\n                valoutput=dr(valimg)\n                val_predict=torch.max(valoutput,dim=1)[1]\n                val_acc=(val_predict==valtarget).sum().item()\n                val_loss=loss_fn(valoutput,valtarget)\n                val_losssum+=val_loss.item()\n                val_accsum+=val_acc\n            #print(f\"这是验证集loss{val_losssum/val_size}\")\n            #print(f\"这是验证集acc{val_accsum/val_size}\")\n            t_loss.append(run_loss/train_size)\n            v_loss.append(val_losssum/val_size)\n            t_acc.append(train_acc/train_size)\n            v_acc.append(val_accsum/val_size)\n    \n        if best_acc<(val_accsum/val_size):\n            torch.save(dr,\"/kaggle/working//bestone1.pth\")\n    history = {'train_loss': t_loss, 'val_loss': v_loss, 'train_acc': t_acc, 'val_acc': v_acc}\n    plot_loss(np.arange(0,epoch), history)\n    plot_acc(np.arange(0,epoch), history)","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:39:45.860308Z","iopub.execute_input":"2023-08-10T12:39:45.861013Z","iopub.status.idle":"2023-08-10T12:52:18.931682Z","shell.execute_reply.started":"2023-08-10T12:39:45.860976Z","shell.execute_reply":"2023-08-10T12:52:18.930641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\nheaders = ['Id','Category']\nl=[]\nwith open('submission.csv','w',encoding='utf8',newline='') as f :\n    writer = csv.writer(f)\n    writer.writerow(headers)\nl=[]\nfor filename in os.listdir(\"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-P1/val/val\"):\n    l.append(filename)\nf=open(\"submission.csv\",\"a\")\nfor line in l:\n    f.write(line+'\\n')\nf.close()","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:52:28.853343Z","iopub.execute_input":"2023-08-10T12:52:28.854043Z","iopub.status.idle":"2023-08-10T12:52:29.004376Z","shell.execute_reply.started":"2023-08-10T12:52:28.854010Z","shell.execute_reply":"2023-08-10T12:52:29.003504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dr=Dr()#这种加载还是需要先实例化一个对象出来\ndr=torch.load(\"/kaggle/working/bestone1.pth\")\ndr.eval()\ndata=pd.read_csv(\"/kaggle/working/submission.csv\")\ndata=data.assign(id=\"\")\ndata=data.assign(Category=\"\")\ndata.drop(data.index[:],inplace=True)\ndata=data.loc[data.index[0:], ['Id', 'Category']]\n\npath = \"/kaggle/input/beyond-visible-spectrum-ai-for-agriculture-P1/val/val\"\nfiles = os.listdir(path)\nnum = len(files)\nfor i in range(num):\n    oldname = files[i]\n    filepath1 = os.path.join(path, files[i])\n    img = skio.imread(filepath1)\n    # filepath,ten=os.path.splitext(filepath1)\n    img = img.transpose(2, 0, 1).astype('float')[:125]\n    img = torch.from_numpy(img)\n    img = img.float()\n    img = img.unsqueeze(0)\n    print(img.shape)\n    pred = dr(img)\n    out = torch.max(pred, dim=1)[1]\n    print(out)\n    if out == 0:\n        newdata = pd.DataFrame({\"Id\": oldname, \"Category\": \"Health\"}, index=[i])\n        \n        data = pd.concat([data, newdata])\n    \n\n    else:\n        if out == 1:\n            newdata = pd.DataFrame({\"Id\": oldname, \"Category\": \"Other\"}, index=[i])\n           \n            data = pd.concat([data, newdata])\n          \n        else:\n            newdata = pd.DataFrame({\"Id\": oldname, \"Category\": \"Rust\"}, index=[i])\n           \n            data = pd.concat([data, newdata])\n            \ndata.to_csv('/kaggle/working/submission.csv',index=None)","metadata":{"execution":{"iopub.status.busy":"2023-08-10T12:53:13.743170Z","iopub.execute_input":"2023-08-10T12:53:13.743598Z","iopub.status.idle":"2023-08-10T12:53:36.574653Z","shell.execute_reply.started":"2023-08-10T12:53:13.743564Z","shell.execute_reply":"2023-08-10T12:53:36.572724Z"},"trusted":true},"execution_count":null,"outputs":[]}]}