{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport json\nimport torch\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom torch.utils.data import ConcatDataset\nfrom torchvision import transforms\nfrom PIL import Image\nimport pytorch_lightning as pl\nimport torchvision\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nfrom sklearn.model_selection import StratifiedKFold\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"a765e293-114b-4a2a-9a77-3c5ddb6af2b5","_cell_guid":"31154160-39de-4ce4-a566-f3faa6cf44a5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:27.418641Z","iopub.execute_input":"2023-06-05T12:10:27.418951Z","iopub.status.idle":"2023-06-05T12:10:44.347380Z","shell.execute_reply.started":"2023-06-05T12:10:27.418924Z","shell.execute_reply":"2023-06-05T12:10:44.346405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nfrom collections import Counter\nfrom tqdm import tqdm\n# define directories\nIMAGE_DIR = '/kaggle/input/benetech-making-graphs-accessible/train/images'\nANNOTATIONS_DIR = '/kaggle/input/benetech-making-graphs-accessible/train/annotations'\nTEST_DIR = '/kaggle/input/benetech-making-graphs-accessible/test/images/'\n# get the list of images\nimage_filenames = os.listdir(IMAGE_DIR)\n\n# json_files = json.load(open(\"/kaggle/input/annotation-data-whole/data.json\"))","metadata":{"_uuid":"ddfa6cb7-7b64-427f-93a4-8fdb842a5295","_cell_guid":"e711734b-01c8-40e6-9479-79de6599c10a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:44.349488Z","iopub.execute_input":"2023-06-05T12:10:44.349793Z","iopub.status.idle":"2023-06-05T12:10:44.877785Z","shell.execute_reply.started":"2023-06-05T12:10:44.349770Z","shell.execute_reply":"2023-06-05T12:10:44.876841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = {}\nfor imgf in tqdm(os.listdir(TEST_DIR)):\n    img_path = os.path.join(TEST_DIR, imgf)\n    base_filename = os.path.splitext(imgf)[0]\n    images[base_filename] = img_path","metadata":{"_uuid":"a92e6865-044f-41a7-9f5b-392573c9b51a","_cell_guid":"d31b6e9f-b477-4bf1-a0fb-ccc8139c9c83","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:44.879092Z","iopub.execute_input":"2023-06-05T12:10:44.879766Z","iopub.status.idle":"2023-06-05T12:10:44.894366Z","shell.execute_reply.started":"2023-06-05T12:10:44.879734Z","shell.execute_reply":"2023-06-05T12:10:44.893317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:10:44.897113Z","iopub.execute_input":"2023-06-05T12:10:44.897754Z","iopub.status.idle":"2023-06-05T12:10:44.904893Z","shell.execute_reply.started":"2023-06-05T12:10:44.897724Z","shell.execute_reply":"2023-06-05T12:10:44.903846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport cv2\ndef plot_confusion_matrix(cm, class_names,title):\n    plt.figure(figsize=(5, 5))\n    sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', cbar=False, xticklabels=class_names, yticklabels=class_names)\n    plt.xlabel('Predicted')\n    plt.ylabel('Actual')\n    plt.title(title)\n    plt.show()\n\ndef pil2cv(image):\n    ''' PIL型 -> OpenCV型 '''\n    return cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)","metadata":{"_uuid":"6c39bc83-b4a9-4e3a-aeb0-4df6d359fee3","_cell_guid":"91ded56f-2d89-4585-8a56-48411f57923e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:44.906998Z","iopub.execute_input":"2023-06-05T12:10:44.909090Z","iopub.status.idle":"2023-06-05T12:10:44.985554Z","shell.execute_reply.started":"2023-06-05T12:10:44.909065Z","shell.execute_reply":"2023-06-05T12:10:44.984673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestChartDataset(Dataset):\n    def __init__(self, image_paths, transform=None):\n        super().__init__()\n        print(image_paths)\n        self.image_paths = image_paths\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, idx):\n        key = list(self.image_paths.keys())[idx]\n        img_path = self.image_paths[key]\n        image = Image.open(img_path)\n        image = pil2cv(image)\n\n        if self.transform:\n            image = self.transform(image=image)[\"image\"]\n\n        return image","metadata":{"_uuid":"53de9409-33ce-456b-aa59-7f7d90b1c226","_cell_guid":"ae4cd80d-375c-4975-a2b2-c7c4ad812e59","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:44.988342Z","iopub.execute_input":"2023-06-05T12:10:44.988926Z","iopub.status.idle":"2023-06-05T12:10:44.995976Z","shell.execute_reply.started":"2023-06-05T12:10:44.988894Z","shell.execute_reply":"2023-06-05T12:10:44.995004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ChartModel(pl.LightningModule):\n    def __init__(self, num_classes):\n        super().__init__()\n        self.num_classes = num_classes\n        self.class_names = ['line', 'dot', 'vertical_bar', 'scatter', 'horizontal_bar']\n        self.save_hyperparameters()\n        self.efficientnet = torchvision.models.efficientnet_b2(torchvision.models.EfficientNet_B2_Weights.DEFAULT)\n        self.lr = 3e-4\n        self.efficientnet.classifier[1] = torch.nn.Linear(self.efficientnet.classifier[1].in_features, num_classes)\n        self.loss = torch.nn.CrossEntropyLoss()\n        self.predictions = []\n        self.targets = []\n        self.sources = []\n        self.best_val_loss=9999\n        self.epochs_since_best_loss =0\n        self.unfrozen_after = 7\n        self.unfreeze_layer()\n    def forward(self, x):\n        return self.efficientnet(x)\n\n    def training_step(self, batch, batch_idx):\n        x, y,_ = batch\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('train_loss', loss)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        x, y,_ = batch\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('val_loss', loss)\n        return loss\n\n    def test_step(self, batch, batch_idx):\n        x, y, source = batch\n        source = np.array(source)\n        y_hat = self(x)\n        loss = self.loss(y_hat, y)\n        self.log('test_loss', loss)\n        \n        # log metrics for different sources separately\n        preds = torch.argmax(y_hat, dim=1)\n        \n        self.predictions.append(preds)\n        self.targets.append(y)\n        self.sources.append(source)\n    def on_test_epoch_end(self):\n        preds = torch.cat(self.predictions).cpu().numpy()\n        targets = torch.cat(self.targets).cpu().numpy()\n        sources = np.concatenate(self.sources)\n        print(\"accuracy: \",accuracy_score(targets,preds))\n        if (sources==\"generated\").sum()>0:\n            cm = confusion_matrix(targets[sources==\"generated\"], preds[sources==\"generated\"],labels=range(self.num_classes))\n            plot_confusion_matrix(cm, self.class_names,\"generated\")\n        if (sources==\"extracted\").sum()>0:\n            cm = confusion_matrix(targets[sources==\"extracted\"], preds[sources==\"extracted\"],labels=range(self.num_classes))\n            plot_confusion_matrix(cm, self.class_names,\"extracted\")\n    def configure_optimizers(self):\n        return torch.optim.Adam(self.parameters(),lr=self.lr)\n    \n    def on_validation_epoch_end(self):\n        current_val_loss = self.trainer.callback_metrics['val_loss']\n        if current_val_loss < self.best_val_loss:\n            self.best_val_loss = current_val_loss\n            self.epochs_since_best_loss = 0\n        else:\n            self.epochs_since_best_loss += 1\n\n        if self.epochs_since_best_loss >= 5:\n            self.unfrozen_after -=1\n            self.unfrozen_after = max(self.unfrozen_after,0)\n            print(\"Unfreezing layer: \",self.unfrozen_after)\n            \n            self.unfreeze_layer()\n            self.lr /= 2\n            self.epochs_since_best_loss = 0\n            self.trainer.optimizers[0] = self.configure_optimizers()\n            \n    def unfreeze_layer(self):\n        reqgrad=False\n        for name,param in self.efficientnet.named_parameters():\n            if f\"features.{self.unfrozen_after}\" in name:\n                reqgrad = True\n            param.requires_grad = reqgrad","metadata":{"_uuid":"68cc757e-b754-4ae7-a9e6-0f5f4c388a78","_cell_guid":"b3647fce-d862-455c-815c-d8cfa055011d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:44.997584Z","iopub.execute_input":"2023-06-05T12:10:44.997915Z","iopub.status.idle":"2023-06-05T12:10:45.019327Z","shell.execute_reply.started":"2023-06-05T12:10:44.997886Z","shell.execute_reply":"2023-06-05T12:10:45.018442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5)","metadata":{"_uuid":"c8683630-b22d-4ac5-a12d-f0b1b77d79c8","_cell_guid":"a2e56aa6-6950-45eb-8ace-fbbb93febe02","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:45.020752Z","iopub.execute_input":"2023-06-05T12:10:45.021080Z","iopub.status.idle":"2023-06-05T12:10:45.037089Z","shell.execute_reply.started":"2023-06-05T12:10:45.021050Z","shell.execute_reply":"2023-06-05T12:10:45.036079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUGMENTATIONS = [\n                    A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1, always_apply=False, p=0.5),\n                    A.OneOf([\n                        A.GaussNoise(var_limit=(10.0, 20.0), mean=0, per_channel=True, always_apply=False, p=0.5),\n                        A.ISONoise(color_shift=(0.01, 0.05), intensity=(0.1, 0.5), always_apply=False, p=0.5),\n                    ], p=0.5),\n                    A.PixelDropout(dropout_prob=0.01, drop_value=None, always_apply=False, p=0.5),\n                    A.ToGray(p=0.5),\n                    A.RandomScale(scale_limit=(-0.5,0)),\n                    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.5),\n                    A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n                    A.OneOf([\n                        A.MotionBlur(blur_limit=5, p=0.5),\n                        A.MedianBlur(blur_limit=5, p=0.5),\n                    ], p=0.5),\n]\nALWAYS_APPLY = [\n                    A.Resize(288,288,p=1.0),\n                    A.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225),p=1.0),                  \n                    ToTensorV2()\n]","metadata":{"_uuid":"cf9a2c71-0c85-4093-a88f-bfc20cd27d3e","_cell_guid":"5d6fc2ab-f584-4a7b-88c3-947889b11e0a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:45.038704Z","iopub.execute_input":"2023-06-05T12:10:45.039045Z","iopub.status.idle":"2023-06-05T12:10:45.051153Z","shell.execute_reply.started":"2023-06-05T12:10:45.039016Z","shell.execute_reply":"2023-06-05T12:10:45.050196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = TestChartDataset(image_paths=images, transform=A.Compose(ALWAYS_APPLY))\ntest_loader = DataLoader(test_set, batch_size=32)\n\nnum_classes = 5\n\npredictions = []\n\nfor fold in range(5):\n    model = ChartModel(num_classes)\n    model.load_from_checkpoint(f'/kaggle/input/99-7-accuracy-efficientnet-b2-chart-classify/checkpoints/fold0_chart-model-epoch=03-val_loss=0.01.ckpt')\n    model.eval()\n    fold_predictions = []\n    for batch in test_loader:\n        images_path = batch\n        outputs = model(images_path)\n\n        _, predicted_labels = torch.max(outputs, 1)\n        fold_predictions.extend(predicted_labels.tolist())\n\n    predictions.append(fold_predictions)\n\nfinal_predictions = []\nfor i in range(len(images)):\n    labels = [fold_predictions[i] for fold_predictions in predictions]\n    majority_label = max(set(labels), key=labels.count)\n    final_predictions.append(majority_label)\n\nprint(\"Final Predictions:\", final_predictions)","metadata":{"_uuid":"3fd11d1a-a8b3-42f8-831d-3e514ee0e298","_cell_guid":"10019f23-eda9-4dc4-8481-d8f38c9a1844","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-06-05T12:10:56.669053Z","iopub.execute_input":"2023-06-05T12:10:56.669433Z","iopub.status.idle":"2023-06-05T12:11:10.318411Z","shell.execute_reply.started":"2023-06-05T12:10:56.669398Z","shell.execute_reply":"2023-06-05T12:11:10.317431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}