{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import timm\nfrom pprint import pprint\npprint(timm.list_models('efficientnet%'))","metadata":{"execution":{"iopub.status.busy":"2023-08-01T08:35:31.597085Z","iopub.execute_input":"2023-08-01T08:35:31.597440Z","iopub.status.idle":"2023-08-01T08:35:31.603524Z","shell.execute_reply.started":"2023-08-01T08:35:31.597413Z","shell.execute_reply":"2023-08-01T08:35:31.602496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model1 = timm.create_model(model_name = 'efficientnet_b5')","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:25.947910Z","iopub.execute_input":"2023-08-01T07:03:25.948388Z","iopub.status.idle":"2023-08-01T07:03:26.697850Z","shell.execute_reply.started":"2023-08-01T07:03:25.948346Z","shell.execute_reply":"2023-08-01T07:03:26.696646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport os\nfrom datetime import datetime\nimport time\nimport random\nimport cv2\nimport pandas as pd\nimport numpy as np\nimport albumentations as A\nimport matplotlib.pyplot as plt\nfrom albumentations.pytorch.transforms import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch.utils.data import Dataset,DataLoader\nfrom torch.utils.data.sampler import SequentialSampler, RandomSampler\nfrom glob import glob","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:26.699358Z","iopub.execute_input":"2023-08-01T07:03:26.699905Z","iopub.status.idle":"2023-08-01T07:03:28.418953Z","shell.execute_reply.started":"2023-08-01T07:03:26.699873Z","shell.execute_reply":"2023-08-01T07:03:28.417783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEED = 42\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:28.422055Z","iopub.execute_input":"2023-08-01T07:03:28.422448Z","iopub.status.idle":"2023-08-01T07:03:28.433677Z","shell.execute_reply.started":"2023-08-01T07:03:28.422413Z","shell.execute_reply":"2023-08-01T07:03:28.432534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Configs = {\n    \n}","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:28.435186Z","iopub.execute_input":"2023-08-01T07:03:28.435687Z","iopub.status.idle":"2023-08-01T07:03:28.441525Z","shell.execute_reply.started":"2023-08-01T07:03:28.435646Z","shell.execute_reply":"2023-08-01T07:03:28.440350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking = pd.read_csv('../input/global-wheat-detection/train.csv')\nmarking.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:28.442791Z","iopub.execute_input":"2023-08-01T07:03:28.443188Z","iopub.status.idle":"2023-08-01T07:03:28.767956Z","shell.execute_reply.started":"2023-08-01T07:03:28.443151Z","shell.execute_reply":"2023-08-01T07:03:28.766956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))\nprint(bboxs.shape)\nbboxs[:5]","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:28.769478Z","iopub.execute_input":"2023-08-01T07:03:28.769822Z","iopub.status.idle":"2023-08-01T07:03:29.296767Z","shell.execute_reply.started":"2023-08-01T07:03:28.769793Z","shell.execute_reply":"2023-08-01T07:03:29.295568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, column in enumerate(['x', 'y', 'w', 'h']):\n    marking[column] = bboxs[:,i]\nmarking.drop(columns=['bbox'], inplace=True)\nmarking.head()","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:29.298092Z","iopub.execute_input":"2023-08-01T07:03:29.298790Z","iopub.status.idle":"2023-08-01T07:03:29.329097Z","shell.execute_reply.started":"2023-08-01T07:03:29.298756Z","shell.execute_reply":"2023-08-01T07:03:29.327912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## K折交叉验证","metadata":{}},{"cell_type":"code","source":"marking.plot(x = 3, y = 1, kind='hist')","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:29.330608Z","iopub.execute_input":"2023-08-01T07:03:29.331092Z","iopub.status.idle":"2023-08-01T07:03:29.664425Z","shell.execute_reply.started":"2023-08-01T07:03:29.331013Z","shell.execute_reply":"2023-08-01T07:03:29.663293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"marking['source'].hist(bins=15);","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:29.665989Z","iopub.execute_input":"2023-08-01T07:03:29.667010Z","iopub.status.idle":"2023-08-01T07:03:29.918521Z","shell.execute_reply.started":"2023-08-01T07:03:29.666969Z","shell.execute_reply":"2023-08-01T07:03:29.917552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n\ndf_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count()\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\n\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:03:29.919701Z","iopub.execute_input":"2023-08-01T07:03:29.919994Z","iopub.status.idle":"2023-08-01T07:03:30.324093Z","shell.execute_reply.started":"2023-08-01T07:03:29.919968Z","shell.execute_reply":"2023-08-01T07:03:30.322465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_folds = marking[['image_id']].copy()\ndf_folds.loc[:, 'bbox_count'] = 1\ndf_folds = df_folds.groupby('image_id').count() # 分组求和\ndf_folds.loc[:, 'source'] = marking[['image_id', 'source']].groupby('image_id').min()['source']\ndf_folds.loc[:, 'stratify_group'] = np.char.add(\n    df_folds['source'].values.astype(str),\n    df_folds['bbox_count'].apply(lambda x: f'_{x // 15}').values.astype(str)\n)\ndf_folds.loc[:, 'fold'] = 0\nfor fold_number, (train_index, val_index) in enumerate(skf.split(X=df_folds.index, y=df_folds['stratify_group'])):\n    df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number\ndf_folds[df_folds['fold']==1]\n","metadata":{"execution":{"iopub.status.busy":"2023-08-01T07:33:05.373250Z","iopub.execute_input":"2023-08-01T07:33:05.373616Z","iopub.status.idle":"2023-08-01T07:33:05.774002Z","shell.execute_reply.started":"2023-08-01T07:33:05.373588Z","shell.execute_reply":"2023-08-01T07:33:05.772933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## data augement","metadata":{}},{"cell_type":"code","source":"# Albumentations\ndef get_train_transform():\n    return A.Compose([\n        A.Flip(0.5),\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})\n\ndef get_valid_transform():\n    return A.Compose([\n        ToTensorV2(p=1.0)\n    ], bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']})","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_ROOT_PATH = '../input/global-wheat-detection/train'\n\nclass DatasetRetriever(Dataset):\n\n    def __init__(self, marking, image_ids, transforms=None, test=False):\n        super().__init__()\n\n        self.image_ids = image_ids\n        self.marking = marking\n        self.transforms = transforms\n        self.test = test\n\n    def __getitem__(self, index: int):\n        image_id = self.image_ids[index]\n        \n        if self.test or random.random() > 0.5:\n            image, boxes = self.load_image_and_boxes(index)\n        else:\n            image, boxes = self.load_cutmix_image_and_boxes(index)\n\n        # there is only one class\n        labels = torch.ones((boxes.shape[0],), dtype=torch.int64)\n        \n        target = {}\n        target['boxes'] = boxes\n        target['labels'] = labels\n        target['image_id'] = torch.tensor([index])\n\n        if self.transforms:\n            for i in range(10):\n                sample = self.transforms(**{\n                    'image': image,\n                    'bboxes': target['boxes'],\n                    'labels': labels\n                })\n                if len(sample['bboxes']) > 0:\n                    image = sample['image']\n                    target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)\n                    target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]  #yxyx: be warning\n                    break\n\n        return image, target, image_id\n\n    def __len__(self) -> int:\n        return self.image_ids.shape[0]\n\n    def load_image_and_boxes(self, index):\n        image_id = self.image_ids[index]\n        image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)\n        image /= 255.0\n        records = self.marking[self.marking['image_id'] == image_id]\n        boxes = records[['x', 'y', 'w', 'h']].values\n        boxes[:, 2] = boxes[:, 0] + boxes[:, 2]\n        boxes[:, 3] = boxes[:, 1] + boxes[:, 3]\n        return image, boxes\n\n    def load_cutmix_image_and_boxes(self, index, imsize=1024):\n        \"\"\" \n        This implementation of cutmix author:  https://www.kaggle.com/nvnnghia \n        Refactoring and adaptation: https://www.kaggle.com/shonenkov\n        \"\"\"\n        w, h = imsize, imsize\n        s = imsize // 2\n    \n        xc, yc = [int(random.uniform(imsize * 0.25, imsize * 0.75)) for _ in range(2)]  # center x, y\n        indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1) for _ in range(3)]\n\n        result_image = np.full((imsize, imsize, 3), 1, dtype=np.float32)\n        result_boxes = []\n\n        for i, index in enumerate(indexes):\n            image, boxes = self.load_image_and_boxes(index)\n            if i == 0:\n                x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc  # xmin, ymin, xmax, ymax (large image)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h  # xmin, ymin, xmax, ymax (small image)\n            elif i == 1:  # top right\n                x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc\n                x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h\n            elif i == 2:  # bottom left\n                x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)\n            elif i == 3:  # bottom right\n                x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)\n                x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)\n            result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]\n            padw = x1a - x1b\n            padh = y1a - y1b\n\n            boxes[:, 0] += padw\n            boxes[:, 1] += padh\n            boxes[:, 2] += padw\n            boxes[:, 3] += padh\n\n            result_boxes.append(boxes)\n\n        result_boxes = np.concatenate(result_boxes, 0)\n        np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])\n        result_boxes = result_boxes.astype(np.int32)\n        result_boxes = result_boxes[np.where((result_boxes[:,2]-result_boxes[:,0])*(result_boxes[:,3]-result_boxes[:,1]) > 0)]\n        return result_image, result_boxes","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = DatasetRetriever(\n    image_ids=df_folds[df_folds['fold'] != fold_number].index.values,\n    marking=marking,\n    transforms=get_train_transforms(),\n    test=False,\n)\n\nvalidation_dataset = DatasetRetriever(\n    image_ids=df_folds[df_folds['fold'] == fold_number].index.values,\n    marking=marking,\n    transforms=get_valid_transforms(),\n    test=True,\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" def collate_fn(batch):\n    return tuple(zip(*batch))\n\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset,\n    batch_size=TrainGlobalConfig.batch_size,\n    sampler=RandomSampler(train_dataset),\n    pin_memory=False,\n    drop_last=True,\n    num_workers=TrainGlobalConfig.num_workers,\n    collate_fn=collate_fn,\n)\nval_loader = torch.utils.data.DataLoader(\n    validation_dataset, \n    batch_size=TrainGlobalConfig.batch_size,\n    num_workers=TrainGlobalConfig.num_workers,\n    shuffle=False,\n    sampler=SequentialSampler(validation_dataset),\n    pin_memory=False,\n    collate_fn=collate_fn,\n)","metadata":{},"execution_count":null,"outputs":[]}]}