{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### 感谢Xichao Wang提供的baseline，本实验基于此baseline稍作调整，在下面做简单陈列：\n- 模型：使用tf_efficientnet_l2_ns_475\n- 数据：参考了Xichao Wang的方法，对train做了数据清洗，删除了部分数据\n- 参数：因为使用了大模型，参数都往小了调\n- 数据增强：选取了Transpose,HorizontalFlip,VerticalFlip,GaussNoise几种基本的数据增强方式，没有使用调色相关的数据增强方法\n\n由于时间关系，有些想法尚未实现，供大家交流：\n- Normalize：本比赛使用Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])，如果针对本数据集计算出对应的各通道均值和标准差，是否会有提升效果。\n- 数据清洗：对Xichao Wang的数据清洗方式进行改进，针对有争议的数据（图片相同，标签不同）予以全部清除\n- 多模型集成\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:08:57.317772Z","iopub.execute_input":"2021-06-29T14:08:57.318165Z","iopub.status.idle":"2021-06-29T14:08:57.324995Z","shell.execute_reply.started":"2021-06-29T14:08:57.318135Z","shell.execute_reply":"2021-06-29T14:08:57.323664Z"}}},{"cell_type":"markdown","source":"## 1.导入库","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"!pip install timm","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:58:59.632048Z","iopub.execute_input":"2021-06-29T14:58:59.632425Z","iopub.status.idle":"2021-06-29T14:59:09.091611Z","shell.execute_reply.started":"2021-06-29T14:58:59.632349Z","shell.execute_reply":"2021-06-29T14:59:09.090633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport time\nfrom glob import glob\nimport random\n\nimport pandas as pd\nimport numpy as np\n\nimport torch\nfrom torch import nn\n\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.cuda.amp import autocast, GradScaler\n\nimport sklearn\nfrom sklearn import preprocessing\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom sklearn.model_selection import StratifiedKFold\n\nimport cv2\n\nfrom tqdm import tqdm\n\nimport timm","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:32.415724Z","iopub.execute_input":"2021-06-29T14:59:32.416090Z","iopub.status.idle":"2021-06-29T14:59:32.422603Z","shell.execute_reply.started":"2021-06-29T14:59:32.416054Z","shell.execute_reply":"2021-06-29T14:59:32.421785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.定义参数","metadata":{}},{"cell_type":"code","source":"CFG = {\n    'rootdir': './classify-leaves/',\n    'fold_num': 5,\n    'seed': 123,\n    'model_arch': 'tf_efficientnet_l2_ns_475',\n    'img_size': 224,\n    'epochs': 8,\n    'train_bs': 12,\n    'valid_bs': 12,\n    'T_0': 4, # Number of iterations for the first restart\n    'lr': 2e-4,\n    'min_lr': 1e-6,\n    'weight_decay':1e-6,\n    'num_workers': 8,\n    'accum_iter': 2, # suppoprt to do batch accumulation for backprop with effectively larger batch size\n    'verbose_step': 1, # 0 (silent),1 (progress bar), 2 (one line per epoch)\n    'device': 'cuda:1'\n}","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:35.769784Z","iopub.execute_input":"2021-06-29T14:59:35.770142Z","iopub.status.idle":"2021-06-29T14:59:35.775568Z","shell.execute_reply.started":"2021-06-29T14:59:35.770112Z","shell.execute_reply":"2021-06-29T14:59:35.774780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.读取数据","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/classify-leaves/train.csv')\ntrain_clean = pd.read_csv('../input/d/baishaothu/classify-leaves/train_clean.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:40.955025Z","iopub.execute_input":"2021-06-29T14:59:40.955378Z","iopub.status.idle":"2021-06-29T14:59:41.021955Z","shell.execute_reply.started":"2021-06-29T14:59:40.955349Z","shell.execute_reply":"2021-06-29T14:59:41.021096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t_labelencoder = preprocessing.LabelEncoder()\nt_labelencoder.fit(train['label'])\ntrain_clean['label'] = t_labelencoder.transform(train_clean['label'])","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:43.166543Z","iopub.execute_input":"2021-06-29T14:59:43.166873Z","iopub.status.idle":"2021-06-29T14:59:43.183611Z","shell.execute_reply.started":"2021-06-29T14:59:43.166843Z","shell.execute_reply":"2021-06-29T14:59:43.182541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_clean","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:45.937670Z","iopub.execute_input":"2021-06-29T14:59:45.938010Z","iopub.status.idle":"2021-06-29T14:59:45.956446Z","shell.execute_reply.started":"2021-06-29T14:59:45.937979Z","shell.execute_reply":"2021-06-29T14:59:45.955300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:49.169954Z","iopub.execute_input":"2021-06-29T14:59:49.170289Z","iopub.status.idle":"2021-06-29T14:59:49.176154Z","shell.execute_reply.started":"2021-06-29T14:59:49.170258Z","shell.execute_reply":"2021-06-29T14:59:49.175236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare Data\nclass MyDataset(Dataset):\n    def __init__(self, df, data_root, transforms=None, output_label=True):\n        \n        super().__init__()\n        self.df = df.reset_index(drop=True).copy()\n        self.transforms = transforms\n        self.data_root = data_root\n        \n        self.output_label = output_label\n        \n        if output_label == True:\n            self.labels = self.df['label'].values\n            \n    def __len__(self):\n        return self.df.shape[0]\n    \n    def __getitem__(self, index: int):\n        # get labels\n        if self.output_label:\n            target = self.labels[index]\n          \n        img  = get_img(\"{}/{}\".format(self.data_root, self.df.loc[index]['image']))\n\n        if self.transforms:\n            img = self.transforms(image=img)['image']\n        \n        if self.output_label == True:\n            return img, target\n        else:\n            return img","metadata":{"execution":{"iopub.status.busy":"2021-06-29T14:59:51.766956Z","iopub.execute_input":"2021-06-29T14:59:51.767293Z","iopub.status.idle":"2021-06-29T14:59:51.774545Z","shell.execute_reply.started":"2021-06-29T14:59:51.767263Z","shell.execute_reply":"2021-06-29T14:59:51.773734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4.数据增强","metadata":{}},{"cell_type":"code","source":"# Augmentation\nfrom albumentations import (\n    HorizontalFlip, VerticalFlip, Rotate, ShiftScaleRotate, RandomBrightnessContrast, Perspective, CLAHE, \n    Transpose, Blur, OpticalDistortion, GridDistortion, HueSaturationValue, ColorJitter, GaussNoise, MotionBlur, MedianBlur,\n    Emboss, Sharpen, Flip, OneOf, SomeOf, Compose, Normalize, CoarseDropout, CenterCrop, GridDropout, Resize\n)\nfrom albumentations.pytorch import ToTensorV2","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_train_transforms():\n    return Compose([\n            OneOf([\n            CoarseDropout(p=0.5),\n            GaussNoise(),\n            ], p=0.5),\n            SomeOf([\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            ShiftScaleRotate(p=0.5),\n            HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            ], n=3, p=0.6),\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.0)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:01:30.242787Z","iopub.execute_input":"2021-06-29T15:01:30.243167Z","iopub.status.idle":"2021-06-29T15:01:30.251315Z","shell.execute_reply.started":"2021-06-29T15:01:30.243137Z","shell.execute_reply":"2021-06-29T15:01:30.250476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_valid_transforms():\n    return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:01:32.120553Z","iopub.execute_input":"2021-06-29T15:01:32.120919Z","iopub.status.idle":"2021-06-29T15:01:32.127504Z","shell.execute_reply.started":"2021-06-29T15:01:32.120886Z","shell.execute_reply":"2021-06-29T15:01:32.126409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_dataloader(df, trn_idx, val_idx, data_root='../input/classify-leaves/'):\n\n    train_ = df.loc[trn_idx,:].reset_index(drop=True)\n    valid_ = df.loc[val_idx,:].reset_index(drop=True)\n        \n    train_ds = MyDataset(train_, data_root, transforms=get_train_transforms(), output_label=True)\n    valid_ds = MyDataset(valid_, data_root, transforms=get_valid_transforms(), output_label=True)\n    \n    train_loader = torch.utils.data.DataLoader(\n        train_ds,\n        batch_size=CFG['train_bs'],\n        pin_memory=True,\n        drop_last=False,\n        shuffle=True,        \n        num_workers=CFG['num_workers'],\n    )\n    val_loader = torch.utils.data.DataLoader(\n        valid_ds, \n        batch_size=CFG['valid_bs'],\n        num_workers=CFG['num_workers'],\n        shuffle=False,\n        pin_memory=True,\n    )\n    return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:01:33.888530Z","iopub.execute_input":"2021-06-29T15:01:33.888926Z","iopub.status.idle":"2021-06-29T15:01:33.897798Z","shell.execute_reply.started":"2021-06-29T15:01:33.888892Z","shell.execute_reply":"2021-06-29T15:01:33.896460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.模型","metadata":{}},{"cell_type":"code","source":"# Model\nclass EfficientModel(nn.Module):\n    def __init__(self, model_arch, n_class, pretrained=False):\n        super().__init__()\n        self.model = timm.create_model(model_arch, pretrained=pretrained)\n        n_features = self.model.classifier.in_features\n        self.model.classifier = nn.Linear(n_features, n_class)\n\n    def forward(self, x):\n        x = self.model(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:28:42.805799Z","iopub.execute_input":"2021-06-29T15:28:42.806211Z","iopub.status.idle":"2021-06-29T15:28:42.812390Z","shell.execute_reply.started":"2021-06-29T15:28:42.806172Z","shell.execute_reply":"2021-06-29T15:28:42.811128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6.训练","metadata":{}},{"cell_type":"code","source":"# Training\ndef train_one_epoch(epoch, model, loss_fn, optimizer, train_loader, device, scheduler=None):\n    model.train()\n\n    running_loss = None\n\n    pbar = tqdm(enumerate(train_loader), total=len(train_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n\n        with autocast():\n            image_preds = model(imgs)\n\n            loss = loss_fn(image_preds, image_labels)\n            scaler.scale(loss).backward()\n\n            if running_loss is None:\n                running_loss = loss.item()\n            else:\n                running_loss = running_loss * .99 + loss.item() * .01\n\n            if ((step + 1) %  CFG['accum_iter'] == 0) or ((step + 1) == len(train_loader)):\n                scaler.step(optimizer)\n                scaler.update()\n                optimizer.zero_grad() \n\n            if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(train_loader)):\n                description = f'epoch {epoch} loss: {running_loss:.4f}'\n                \n                pbar.set_description(description)\n                \n    if scheduler is not None:\n        scheduler.step()\n        \n    return running_loss","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:02:06.152267Z","iopub.execute_input":"2021-06-29T15:02:06.152605Z","iopub.status.idle":"2021-06-29T15:02:06.162235Z","shell.execute_reply.started":"2021-06-29T15:02:06.152578Z","shell.execute_reply":"2021-06-29T15:02:06.161303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None):\n    model.eval()\n\n    loss_sum = 0\n    sample_num = 0\n    image_preds_all = []\n    image_targets_all = []\n    val_loss = None\n    \n    pbar = tqdm(enumerate(val_loader), total=len(val_loader))\n    for step, (imgs, image_labels) in pbar:\n        imgs = imgs.to(device).float()\n        image_labels = image_labels.to(device).long()\n        \n        image_preds = model(imgs)\n        image_preds_all += [torch.argmax(image_preds, 1).detach().cpu().numpy()]\n        image_targets_all += [image_labels.detach().cpu().numpy()]\n        \n        loss = loss_fn(image_preds, image_labels)\n        \n        loss_sum += loss.item()*image_labels.shape[0]\n        sample_num += image_labels.shape[0]\n\n        if ((step + 1) % CFG['verbose_step'] == 0) or ((step + 1) == len(val_loader)):\n            val_loss = loss_sum/sample_num\n            description = f'epoch {epoch} loss: {val_loss:.4f}'\n            pbar.set_description(description)\n    \n    image_preds_all = np.concatenate(image_preds_all)\n    image_targets_all = np.concatenate(image_targets_all)\n    \n    val_acc = (image_preds_all==image_targets_all).mean()\n    print('validation multi-class accuracy = {:.4f}'.format(val_acc))\n    \n    if scheduler is not None:\n        scheduler.step()\n        \n    return val_acc, val_loss","metadata":{"execution":{"iopub.status.busy":"2021-06-29T15:02:08.047593Z","iopub.execute_input":"2021-06-29T15:02:08.047928Z","iopub.status.idle":"2021-06-29T15:02:08.057573Z","shell.execute_reply.started":"2021-06-29T15:02:08.047897Z","shell.execute_reply":"2021-06-29T15:02:08.056399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 7.main","metadata":{}},{"cell_type":"code","source":"if __name__ == '__main__':\n\n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num'], shuffle=True, random_state=CFG['seed']).split(np.arange(train_clean.shape[0]), train_clean.label.values)\n    train_loss_all = []\n    val_loss_all = []\n    val_acc_all = []\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        start_time = time.time()\n        # 测试两个fold\n        # if fold > 1:\n        #     break \n            \n        print('Training with {} started'.format(fold))\n\n        print(len(trn_idx), len(val_idx))\n        train_loader, val_loader = prepare_dataloader(train_clean, trn_idx, val_idx, data_root=CFG['rootdir'])\n\n        device = torch.device(CFG['device'])\n        \n        model = EfficientModel(CFG['model_arch'], train_clean.label.nunique(), pretrained=True).to(device)\n        scaler = GradScaler()\n        optimizer = torch.optim.AdamW(model.parameters(), lr=CFG['lr'], weight_decay=CFG['weight_decay'])\n        scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CFG['T_0'], T_mult=1, eta_min=CFG['min_lr'], last_epoch=-1)\n        #scheduler = torch.optim.lr_scheduler.StepLR(optimizer, gamma=0.1, step_size=CFG['epochs']-1)\n        \n        loss_tr = nn.CrossEntropyLoss().to(device)\n        loss_fn = nn.CrossEntropyLoss().to(device)\n        train_loss_temp = []\n        val_loss_temp = [] \n        val_acc_temp = []\n        \n        for epoch in range(CFG['epochs']):\n            train_loss_temp = train_one_epoch(epoch, model, loss_tr, optimizer, train_loader, device, scheduler=scheduler)\n            train_loss_all.append([fold, epoch, train_loss_temp])\n            with torch.no_grad():\n                val_acc_temp, val_loss_temp = valid_one_epoch(epoch, model, loss_fn, val_loader, device, scheduler=None)\n                \n                val_loss_all.append([fold, epoch, val_loss_temp])\n                val_acc_all.append([fold, epoch, val_acc_temp])\n            if epoch > CFG['epochs'] - 4:  #save last three models\n                torch.save(model.state_dict(),'{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n                \n        del model, optimizer, train_loader, val_loader, scaler, scheduler\n        torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 8.预测","metadata":{}},{"cell_type":"code","source":"# Inference Loop\nCFG['weights'] = [0.8,1,0.9] # weight for out model\nCFG['tta'] = 3 # set TTA times\nCFG['used_epochs'] = [5,6,7] # choose the model\ndef get_inference_transforms():\n    return Compose([\n            Resize(CFG['img_size'], CFG['img_size']),\n            Transpose(p=0.5),\n            HorizontalFlip(p=0.5),\n            VerticalFlip(p=0.5),\n            #HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n            #RandomBrightnessContrast(brightness_limit=(-0.1,0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n            Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], max_pixel_value=255.0, p=1.0),\n            ToTensorV2(p=1.0),\n        ], p=1.)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference_one_epoch(model, data_loader, device):\n    model.eval()\n\n    image_preds_all = []\n    \n    pbar = tqdm(enumerate(data_loader), total=len(data_loader))\n    for step, (imgs) in pbar:\n        imgs = imgs.to(device).float()\n        \n        image_preds = model(imgs)\n        image_preds_all += [torch.softmax(image_preds, 1).detach().cpu().numpy()]\n        \n    image_preds_all = np.concatenate(image_preds_all, axis=0)\n    return image_preds_all","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    \n    seed_everything(CFG['seed'])\n    \n    folds = StratifiedKFold(n_splits=CFG['fold_num']).split(np.arange(train_clean.shape[0]), train_clean.label.values)\n    tst_preds_all = []\n    for fold, (trn_idx, val_idx) in enumerate(folds):\n        print(fold)\n        # if fold > 1:\n        #     break\n\n        print('Inference fold {} started'.format(fold))\n        \n        valid_ = train_clean.loc[val_idx,:].reset_index(drop=True)\n        valid_ds = MyDataset(valid_, CFG['rootdir'], transforms=get_inference_transforms(), output_label=False)\n\n        test = pd.read_csv(CFG['rootdir'] + 'test.csv')\n        test_ds = MyDataset(test, CFG['rootdir'], transforms=get_inference_transforms(), output_label=False)\n\n        val_loader = torch.utils.data.DataLoader(\n            valid_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=True,\n        )\n\n        tst_loader = torch.utils.data.DataLoader(\n            test_ds, \n            batch_size=CFG['valid_bs'],\n            num_workers=CFG['num_workers'],\n            shuffle=False,\n            pin_memory=True,\n        )\n        \n        device = torch.device(CFG['device'])\n        model = EfficientModel(CFG['model_arch'], train.label.nunique()).to(device)\n        \n        val_preds = []\n        tst_preds = []\n        \n        for i, epoch in enumerate(CFG['used_epochs']):    \n            model.load_state_dict(torch.load('{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch)))\n            print('{}_fold_{}_{}'.format(CFG['model_arch'], fold, epoch))\n\n            with torch.no_grad():\n                for _ in range(CFG['tta']):\n                    # val_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, val_loader, device)]\n                    tst_preds += [CFG['weights'][i]/sum(CFG['weights'])/CFG['tta']*inference_one_epoch(model, tst_loader, device)]\n\n                    \n        # val_preds = np.mean(val_preds, axis=0) \n        tst_preds = np.mean(tst_preds, axis=0) \n        tst_preds_all.append(tst_preds)\n        \n        # print('fold {} validation loss = {:.5f}'.format(fold, log_loss(valid_.label.values, val_preds)))\n        # print('fold {} validation accuracy = {:.5f}'.format(fold, (valid_.label.values==np.argmax(val_preds, axis=1)).mean()))\n        del model\n        torch.cuda.empty_cache()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate the average result from all folds\navg_tst = np.mean(tst_preds_all, axis=0)\n\ntest['label'] = np.argmax(avg_tst, axis=1)\ntest.head()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# inverse our label\ntest['label'] = t_labelencoder.inverse_transform(test['label'])\n\ntest.to_csv('submission.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## 9.成绩","metadata":{}},{"cell_type":"code","source":"Private Score：0.98636   18th\nPublic  Score：0.98295   14th","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}