{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# About this notebook \n", "\n", "TBD...\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Directory settings" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Directory settings\n", "# ====================================================\n", "import os\n", "\n", "OUTPUT_DIR = \"./\"\n", "MODEL_DIR = \"../input/cassava-model/\"\n", "if not os.path.exists(OUTPUT_DIR):\n", " os.makedirs(OUTPUT_DIR)\n", "\n", "TRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n", "TEST_PATH = \"../input/cassava-leaf-disease-merged/train\"\n", "# TEST_PATH = \"../input/cassava-leaf-disease-classification/train_images\"\n", "# TEST_PATH = \"../input/cassava-leaf-disease-classification/test_images\"" ] }, { "cell_type": "markdown", "metadata": { "toc-hr-collapsed": true }, "source": [ "# CFG" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# CFG\n", "# ====================================================\n", "class CFG:\n", " debug = False\n", " num_workers = 4\n", " models = [\n", " # \"tf_efficientnet_b3_ns\",\n", " \"tf_efficientnet_b4_ns\",\n", " \"vit_base_patch16_384\",\n", " # \"deit_base_patch16_384\",\n", " \"seresnext50_32x4d\",\n", " ]\n", " size = {\n", " \"tf_efficientnet_b3_ns\": 512,\n", " \"tf_efficientnet_b4_ns\": 512,\n", " \"vit_base_patch16_384\": 384,\n", " \"deit_base_patch16_384\": 384,\n", " \"seresnext50_32x4d\": 512,\n", " }\n", " batch_size = 32\n", " seed = 22\n", " target_size = 5\n", " target_col = \"label\"\n", " n_fold = 5\n", " trn_fold = { # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n", " \"tf_efficientnet_b3_ns\": {\n", " \"best\": [0, 1, 2, 3, 4],\n", " \"final\": [],\n", " },\n", " \"tf_efficientnet_b4_ns\": {\n", " \"best\": [0, 1, 2, 3, 4],\n", " \"final\": [],\n", " },\n", " \"vit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n", " \"deit_base_patch16_384\": {\"best\": [0, 1, 2, 3, 4], \"final\": []},\n", " \"seresnext50_32x4d\": {\"best\": [5, 6, 7, 8, 9], \"final\": []},\n", " }\n", " data_parallel = {\n", " \"tf_efficientnet_b3_ns\": False,\n", " \"tf_efficientnet_b4_ns\": True,\n", " \"vit_base_patch16_384\": False,\n", " \"deit_base_patch16_384\": False,\n", " \"seresnext50_32x4d\": False,\n", " }\n", " weight = {\n", " # \"tf_efficientnet_b3_ns\": None,\n", " \"tf_efficientnet_b4_ns\": 1,\n", " \"vit_base_patch16_384\": 1,\n", " # \"deit_base_patch16_384\": None,\n", " \"seresnext50_32x4d\": 1,\n", " }\n", " tta = 1 # 1: no TTA, >1: TTA\n", " train = False\n", " inference = True" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "weight_sum = sum([CFG.weight[model] for model in CFG.models]) * CFG.tta" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Library" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Library\n", "# ====================================================\n", "import sys\n", "\n", "sys.path.append(\"../input/pytorch-image-models/pytorch-image-models-master\")\n", "\n", "import math\n", "import os\n", "import random\n", "import shutil\n", "import time\n", "import warnings\n", "from collections import Counter, defaultdict\n", "from contextlib import contextmanager\n", "from functools import partial\n", "from pathlib import Path\n", "\n", "import cv2\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", "import scipy as sp\n", "import seaborn as sns\n", "import timm\n", "import torch\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "import torchvision.models as models\n", "from albumentations import (\n", " CenterCrop,\n", " CoarseDropout,\n", " Compose,\n", " Cutout,\n", " HorizontalFlip,\n", " HueSaturationValue,\n", " IAAAdditiveGaussianNoise,\n", " ImageOnlyTransform,\n", " Normalize,\n", " OneOf,\n", " RandomBrightness,\n", " RandomBrightnessContrast,\n", " RandomContrast,\n", " RandomCrop,\n", " RandomResizedCrop,\n", " Resize,\n", " Rotate,\n", " ShiftScaleRotate,\n", " Transpose,\n", " VerticalFlip,\n", ")\n", "from albumentations.pytorch import ToTensorV2\n", "from PIL import Image\n", "from sklearn import preprocessing\n", "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n", "from sklearn.model_selection import StratifiedKFold\n", "from torch.nn.parameter import Parameter\n", "from torch.optim import SGD, Adam\n", "from torch.optim.lr_scheduler import CosineAnnealingLR, CosineAnnealingWarmRestarts, ReduceLROnPlateau\n", "from torch.utils.data import DataLoader, Dataset\n", "from tqdm.auto import tqdm\n", "\n", "warnings.filterwarnings(\"ignore\")\n", "\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Utils" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Utils\n", "# ====================================================\n", "def get_score(y_true, y_pred):\n", " return accuracy_score(y_true, y_pred)\n", "\n", "\n", "@contextmanager\n", "def timer(name):\n", " t0 = time.time()\n", " LOGGER.info(f\"[{name}] start\")\n", " yield\n", " LOGGER.info(f\"[{name}] done in {time.time() - t0:.0f} s.\")\n", "\n", "\n", "def init_logger(log_file=OUTPUT_DIR + \"inference.log\"):\n", " from logging import INFO, FileHandler, Formatter, StreamHandler, getLogger\n", "\n", " logger = getLogger(__name__)\n", " logger.setLevel(INFO)\n", " handler1 = StreamHandler()\n", " handler1.setFormatter(Formatter(\"%(message)s\"))\n", " handler2 = FileHandler(filename=log_file)\n", " handler2.setFormatter(Formatter(\"%(message)s\"))\n", " logger.addHandler(handler1)\n", " logger.addHandler(handler2)\n", " return logger\n", "\n", "\n", "LOGGER = init_logger()\n", "\n", "\n", "def seed_torch(seed=42):\n", " random.seed(seed)\n", " os.environ[\"PYTHONHASHSEED\"] = str(seed)\n", " np.random.seed(seed)\n", " torch.manual_seed(seed)\n", " torch.cuda.manual_seed(seed)\n", " torch.backends.cudnn.deterministic = True\n", "\n", "\n", "seed_torch(seed=CFG.seed)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Data Loading" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
image_idlabelsource
01000015157.jpg02020
11000201771.jpg32020
2100042118.jpg12020
31000723321.jpg12020
41000812911.jpg32020
\n", "
" ], "text/plain": [ " image_id label source\n", "0 1000015157.jpg 0 2020\n", "1 1000201771.jpg 3 2020\n", "2 100042118.jpg 1 2020\n", "3 1000723321.jpg 1 2020\n", "4 1000812911.jpg 3 2020" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/sample_submission.csv\")\n", "# test = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n", "test = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")\n", "test.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Dataset" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Dataset\n", "# ====================================================\n", "class TestDataset(Dataset):\n", " def __init__(self, df, transform=None):\n", " self.df = df\n", " self.file_names = df[\"image_id\"].values\n", " self.transform = transform\n", "\n", " def __len__(self):\n", " return len(self.df)\n", "\n", " def __getitem__(self, idx):\n", " file_name = self.file_names[idx]\n", " file_path = f\"{TEST_PATH}/{file_name}\"\n", " image = cv2.imread(file_path)\n", " image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n", " if self.transform:\n", " augmented = self.transform(image=image)\n", " image = augmented[\"image\"]\n", " return image" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Transforms" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Transforms\n", "# ====================================================\n", "def get_transforms(*, data, size):\n", "\n", " if data == \"train\":\n", " return Compose(\n", " [\n", " # Resize(size, size),\n", " RandomResizedCrop(size, size),\n", " Transpose(p=0.5),\n", " HorizontalFlip(p=0.5),\n", " VerticalFlip(p=0.5),\n", " ShiftScaleRotate(p=0.5),\n", " HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n", " RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n", " CoarseDropout(p=0.5),\n", " Cutout(p=0.5),\n", " Normalize(\n", " mean=[0.485, 0.456, 0.406],\n", " std=[0.229, 0.224, 0.225],\n", " ),\n", " ToTensorV2(),\n", " ]\n", " )\n", "\n", " if data == \"valid\":\n", " return Compose(\n", " [\n", " Resize(size, size),\n", " CenterCrop(size, size),\n", " Normalize(\n", " mean=[0.485, 0.456, 0.406],\n", " std=[0.229, 0.224, 0.225],\n", " ),\n", " ToTensorV2(),\n", " ]\n", " )\n", "\n", " if data == \"inference\":\n", " return Compose(\n", " [\n", " # Resize(size, size),\n", " RandomResizedCrop(size, size),\n", " Transpose(p=0.5),\n", " HorizontalFlip(p=0.5),\n", " VerticalFlip(p=0.5),\n", " # ShiftScaleRotate(p=0.5),\n", " # HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit=0.2, val_shift_limit=0.2, p=0.5),\n", " # RandomBrightnessContrast(brightness_limit=(-0.1, 0.1), contrast_limit=(-0.1, 0.1), p=0.5),\n", " # CoarseDropout(p=0.5),\n", " # Cutout(p=0.5),\n", " Normalize(\n", " mean=[0.485, 0.456, 0.406],\n", " std=[0.229, 0.224, 0.225],\n", " ),\n", " ToTensorV2(),\n", " ]\n", " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# MODEL" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# MODEL\n", "# ====================================================\n", "class CassvaImgClassifier(nn.Module):\n", " def __init__(self, model_name=\"resnext50_32x4d\", pretrained=False):\n", " super().__init__()\n", "\n", " if model_name == \"deit_base_patch16_384\":\n", " # self.model = torch.hub.load(\"facebookresearch/deit:main\", model_name, pretrained=pretrained)\n", " self.model = torch.hub.load(\"../input/fair-deit\", model_name, pretrained=pretrained, source=\"local\")\n", " n_features = self.model.head.in_features\n", " self.model.head = nn.Linear(n_features, CFG.target_size)\n", "\n", " else:\n", " self.model = timm.create_model(model_name, pretrained=pretrained)\n", "\n", " if \"resnext50_32x4d\" in model_name:\n", " n_features = self.model.fc.in_features\n", " self.model.fc = nn.Linear(n_features, CFG.target_size)\n", "\n", " elif model_name.startswith(\"tf_efficientnet\"):\n", " n_features = self.model.classifier.in_features\n", " self.model.classifier = nn.Linear(n_features, CFG.target_size)\n", "\n", " elif model_name.startswith(\"vit_\"):\n", " n_features = self.model.head.in_features\n", " self.model.head = nn.Linear(n_features, CFG.target_size)\n", "\n", " def forward(self, x):\n", " x = self.model(x)\n", " return x" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Helper functions" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# ====================================================\n", "# Helper functions\n", "# ====================================================\n", "def inference(model, states, test_loader, device, data_parallel):\n", " model.to(device)\n", "\n", " # Use multi GPU\n", " if device == torch.device(\"cuda\") and data_parallel:\n", " model = torch.nn.DataParallel(model) # make parallel\n", " # torch.backends.cudnn.benchmark=True\n", "\n", " tk0 = tqdm(enumerate(test_loader), total=len(test_loader))\n", " probs = []\n", " for i, (images) in tk0:\n", " images = images.to(device)\n", " avg_preds = []\n", " for state in states:\n", " model.load_state_dict(state[\"model\"])\n", " model.eval()\n", " with torch.no_grad():\n", " y_preds = model(images)\n", " avg_preds.append(y_preds.softmax(1).to(\"cpu\").numpy())\n", " avg_preds = np.mean(avg_preds, axis=0)\n", " probs.append(avg_preds)\n", " probs = np.concatenate(probs)\n", " return probs" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# inference" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "========== Model: tf_efficientnet_b4_ns, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "519117ae6e3f433caedbcedd08d50ab4", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Inference example: [0.4113303 0.16465071 0.19031808 0.01710353 0.21659732]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "========== Model: vit_base_patch16_384, TTA: 0, Saved: best, Fold: [0, 1, 2, 3, 4] ==========\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7075695aa1d044c3900a5e1bc761fe7e", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Inference example: [0.44151726 0.12938017 0.24077013 0.04045077 0.14788169]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "========== Model: seresnext50_32x4d, TTA: 0, Saved: best, Fold: [5, 6, 7, 8, 9] ==========\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "c5381a693b0f4c7d9c5a5be73f68b76a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=824.0), HTML(value='')))" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Inference example: [0.2990355 0.3897496 0.09844354 0.02557188 0.1871995 ]\n", "========== Overall ==========\n", "Submission example: [0.38396105 0.2279268 0.17651057 0.02770873 0.18389285]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
image_idlabelsource
01000015157.jpg02020
11000201771.jpg32020
2100042118.jpg42020
31000723321.jpg12020
41000812911.jpg32020
\n", "
" ], "text/plain": [ " image_id label source\n", "0 1000015157.jpg 0 2020\n", "1 1000201771.jpg 3 2020\n", "2 100042118.jpg 4 2020\n", "3 1000723321.jpg 1 2020\n", "4 1000812911.jpg 3 2020" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ====================================================\n", "# inference\n", "# ====================================================\n", "predictions = None\n", "for model_name in CFG.models:\n", " for i in range(CFG.tta):\n", " model = CassvaImgClassifier(model_name, pretrained=False)\n", " states = []\n", " for saved_model in [\"best\", \"final\"]:\n", " if CFG.trn_fold[model_name][saved_model] != []:\n", " LOGGER.info(\n", " f\"========== Model: {model_name}, TTA: {i}, Saved: {saved_model}, Fold: {CFG.trn_fold[model_name][saved_model]} ==========\"\n", " )\n", " states += [\n", " torch.load(MODEL_DIR + f\"{model_name}_fold{fold}_{saved_model}.pth\")\n", " for fold in CFG.trn_fold[model_name][saved_model]\n", " ]\n", "\n", " if i == 0: # no TTA\n", " test_dataset = TestDataset(test, transform=get_transforms(data=\"valid\", size=CFG.size[model_name]))\n", " else:\n", " test_dataset = TestDataset(test, transform=get_transforms(data=\"inference\", size=CFG.size[model_name]))\n", "\n", " test_loader = DataLoader(\n", " test_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=CFG.num_workers, pin_memory=True\n", " )\n", "\n", " inf = inference(model, states, test_loader, device, CFG.data_parallel[model_name])\n", "\n", " LOGGER.info(f\"Inference example: {inf[0]}\")\n", "\n", " if predictions is None:\n", " predictions = inf[np.newaxis] * CFG.weight[model_name]\n", " else:\n", " predictions = np.append(predictions, inf[np.newaxis] * CFG.weight[model_name], axis=0)\n", "\n", "sub = np.sum(predictions, axis=0) / weight_sum\n", "LOGGER.info(f\"========== Overall ==========\")\n", "LOGGER.info(f\"Submission example: {sub[0]}\")\n", "np.savetxt(f\"{OUTPUT_DIR}/predictions.csv\", sub, delimiter=\",\")\n", "\n", "# submission\n", "test[\"label\"] = sub.argmax(1)\n", "test[[\"image_id\", \"label\"]].to_csv(OUTPUT_DIR + \"submission.csv\", index=False)\n", "test.head()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Validation" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "# train = pd.read_csv(\"../input/cassava-leaf-disease-classification/train.csv\")\n", "train = pd.read_csv(\"../input/cassava-leaf-disease-merged/merged.csv\")" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "cm = confusion_matrix(train[\"label\"], test[\"label\"])\n", "cm = cm / cm.astype(np.float).sum(axis=1)" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAAD4CAYAAABPLjVeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA9+0lEQVR4nO3dd3gU1frA8e9JoycBJBsgAWlKV6RYEFIghBJS6KDgVaoFUemCqCBVughSLfdaLkgLJCT0KvVaAAWkSEkgG0RCN2Vzfn9sDNkkkADJ7rK/98Mzz5PZOTv7vszsu2fPzM4orTVCCCGsw8nWAQghxP8nUnSFEMKKpOgKIYQVSdEVQggrkqIrhBBW5FLYL1Cm5zcOd3rEucXdbR1CoVDK1hEUPCdHTApISUu3dQiFwr2o0wNvsGIN3sh3zbn10xyr7yDS0xVCCCsq9J6uEEJYlbLvvqQUXSGEY3FytnUEdyVFVwjhWOx8HF+KrhDCscjwghBCWJH0dIUQwoqkpyuEEFYkPV0hhLAiOXtBCCGsSIYXhBDCimR4QQghrEh6ukIIYUVSdIUQwoqc5UCaEEJYj4zpCiGEFcnwghBCWJH0dAtGi3rlmdCzIc5Oin9vPcmstb9ZLK9Ytjhz+z2LR3FXnJ0UHy79hY2/nMe/rjdjujyJm4sTKWnpvP/dT+z4zWijLCzt2rmDqZPHYzKlE9GhEy/36WexPCUlhffeHc6R337F09OTSR9Pp0JFH/b8sIvZM6eRlpqKi6srbw0eRpOnn7FRFma7du7g40njSTelE96xE6/klstIcy4enp5MnmrOJSnpMkPfHsSvhw8TGh7OiFFjMp/zev8+XLx4EZPJRIOnGjJy9BicC3m8bteO7UzOyCOiY2d6982Zx6iRwzjyqzmPKdNmULGiDwCLF85n5fLvcXJ2YvjI0TR9vhnJycm83OsFUlNSSDOZCGoVzGtvvAmA1po5s2eyPjYGZ2cnOnftzgsv9irU/AB+2LWDaZMnkJ6eTlhEJ/7Vu2+OHN8fNZyjR37Dw8OTCVOmU6FiRX49dJDx494nI3j6DnidgBZBnD79B+8Oeyfz+efjztHvtYH0ePGlQs8lV9LTfXBOSjHlpUZ0mLyZ83/dYtPYYGJ+jOPY+auZbYaE1WXVvjN8vukEj1dw579D/HnynUguXUumx/RtJCTdopaPB8uGBlB30CrbJZPBZDIxefxY5i5YgsHbwIvdOuMXEEjVatUz26xa8T3u7u5ERq8ndl0Us2ZMY/LUGXiWLs2sOfMo52XgxPHfeX1AH2I3bbdpLpM+Gsu8heZcXuhqzqVatlxKubsTuW49MdFRzJo+jcnTZlDErQivDRzEiePHOXnid4v1Tp42k5IlS6K1Zsjbb7IhNobWbdsVah4Txo9l/sLPMRgM9OjaCf+AQKpVv53HyuXLcHd3Z23MBtZFRzFz+lQ+njaTkydOEBMdxYrIKBITjfTv8zKRUbG4ubmxaMmXFC9RgtTUVP7VswfPN2tO/SeeZPWqFSQkXGD12nU4OTlx6dKlQssta45TJoxjzvzFGAwGXurRheb+ARb73eqV3+Pu7sHKtbGsXxfFJzOnMvHjGVSrXoOvvlmGi4sLf15MpEfnCJr5BfDoo1X4ZunKzPW3DfInILBloedyR3be083zI0EpVVMpNVwpNTtjGq6UqmWN4P7RsFpZ/jBe58zFG6Sa0lmx5wxtGvpYtNFaU6qoKwDuxd1ISLoFwKEzlzP/PhJ3hWJuzri52P6T8PChg/hUqoSPry+urm4Et2nL1i2bLNps3bKJkNBwAFoEBbN/72601tSsVZtyXgYAqlWvQfLfyaSkpFg7hUyHDx3EN3sum7PlsnkT7cPCAWjZKph9GbkUK16cBk81pEgRtxzrLVmyJABpaWmkpaaiCvnNdPjQQXx9K5vzcHOjddt2ObbJls2bCQ2LACCoVTD79pjz2LplE63btsPNzQ0fH198fStz+NBBlFIUL1Hidh5paZlFYel339J/wOs4OZn3x7JlyxZqfgC/Hj6Ir28lfHzM2yqodVu2bd1s0Wb7ls20Cw0DIDAomP379qC1pmixYri4mPtpyckpuW6P/Xv34OPrS/kKFQs9lztycs7/ZIvw7rZQKTUc+A5QwL6MSQHfKqVGFH54ZuVLFyP+rxuZ8+f/ukn50sUt2kxecYguTatweFY4/x3iz/CvDuRYT2hjX345fdkubup3MdGIt3f5zHkvgzeJRmO2NomZbVxcXChZshRJSUkWbTZtiKVmrdq4ueUsWtaSmGjEkCUXg8Gbi4nGbG3yziU3r/XrTQu/phQvUYKWrYILNO7sEo1GvMt7Z857GQwYjdnzMFrmUaoUSUmXMRqNGLxvP9fgbcjcniaTiS4dwgho9hzPPPsc9es/AUDcuXPExkTTvUsHXuvfhzNnThdqfmDepyzi9DJwMZccDdm21ZWMbXX44C90iQihe6cwRox+P7MI/2N9TDTBrQvv20i+KKf8TzaQ16v2BhprrSdprf+TMU0CmmQsy5VSqp9S6oBS6kDy8c13alagOj77KN/uOEXdQavoOnUrnw14zuJbRs2KHrzf9Une+XyfVeKxhpMnjjN7xjRGvf+hrUMpNHMXLGbDlh2kpKSwf+8eW4dzX5ydnVm6YjXrN2/j8KGDHD9uHkZJSUnBrUgRvl26gg6duvD+6HdtHGne6tZ/gqUr1/LlN0v5YvFCkpOTM5elpqawfdtmWhTyh2OelMr/ZAN5Fd10oEIuj5fPWJYrrfUCrXUjrXWjIjUCHyQ+AC5cvkXFMiUy5yuUKc6Fyzct2rzoV5VVe88CsP/EnxRxdaZsqSLm9qWL8dWgZrw2fzenE68/cDwFoZyXgYSEC5nzicYEvAyGbG28MtukpaVx/fo1PD09ATAmJDD4rTcYO2Eyvr6VrBZ3bry8DBiz5GI0JmQOf9xuc+dc8lKkSBH8A1rk+Kpf0LwMBhIuJGTOJxqNGAzZ8zBY5nHtGp6epTEYDBgTbj/XmGDMsT3d3d1p3ORpfti5AzD3hlu0DAKgRcsgjv9+rFDyyqqcl5dlnIlGyuWSozHbtvLItq2qVK1G8eLFOXnieOZjP+zcQc2atSlb9pHCSyA/HvKe7lvAJqXUOqXUgowpBtgEDCr06DL8eOoSVb1LUalcCVydnejwTGVifoy3aBN36SbN65h3nscquFPE1Yk/rybjXtyV74b4M3bpz+w9/qe1Qs5Tnbr1OHfmDPFxcaSmphC7Lho/f8sPKD//QNZGrgLMwwiNmzyDUoprV6/y5uv9GfjWYJ5s8JQNordUp249zp61zMU/IFsuAYGsWb0KgI3rY2n89DN3HaO9efMGFy8mAuY3/s7t23i0StVCywH+yeM0cXHnSE1JISY6Cr9sefgHBBK52nzQaMP6WJpk5OEXEEhMdBQpKSnExZ3j7NnT1K1Xn7/++ourV80HfP/++2/27P4hM4+AwJbs37cXgAP791G58qOFmh9A7TqW22pDTDTN/QIs2jTzDyAqcjUAm7Psd/FxceYxaeDC+XhOnz5FhSxjt7HromjVxsZDC2D3RVdpre/eQCknzMMJ//zvxgP7tdam/LxAmZ7f3P0F8qnlExWY8MJTODspvt5+iumRvzKyQz1++uMvYn6K5/EK7szs/TQlirqgNXzw3U9sOZzA4LA6vNW+DqcSrmWuq+OUzfx5Nfkur3Z35xZ3L4iU2Ll9G1OnTCDdlE5oREf69BvAvDmzqV2nLn4BgSQnJ/PeyGEcPXoEDw8PJk6Zjo+vL4vmz2PJ4gVUqlQ5c11z5y+mzAMeiHmQb1s7tm9j6mRzLmERHenTfwBzM3Lxz8hl9MhhHDtyBHcPDyZ9bM4FoG2rQG5cv0Fqaiql3Esxd8FiPD08efP1AaSmpJCuNY2aNGHIsJE5xhDz4nSPSe3Yvo0pkyaQnm4iPKIjffu/yqefzKJOnbr4B7YgOTmZUSOGcjQjjylTZ2TmsXD+PFatXI6zszPDRrzL8838+P3YUUa/O4L0dBPp6ZpWwa0Z8NobAFy9epV3hw/hwoULFC9enNFjPuTxmjXzFeeDHJfYtWMb06dMxJSeTmh4B17pO4DPPp1NrTp18fM3b6v3Rw3n2NEjuLt7MH7KNHx8fIles5ovlizExdUVJ6Xo0/81/DPOUrh18ybtWweyKmoDJUuVuu/Y3Is6PfB3/mJh8/Ndc26t7m/1MYY8i+6DKqiia08KqujaGzs/0+a+3GvRfVjYw8HgwlAgRTd8Qf6L7qp+Vt9BHorzdIUQIt/kxxFCCGFFdv7tRoquEMKhFPaPaB6UFF0hhEORoiuEEFakHvxYXKGSoiuEcCjS0xVCCCuSoiuEEFYkRVcIIazJvmuuFF0hhGORnq4QQljRPxeFt1dSdIUQDsXee7r2/ZEghBD3St3DlNeqlGqtlDqmlDqR291ylFKVlFJblFI/KaUOKqXa5rVOKbpCCIeilMr3lMd6nIFPgTZAbaC7Uqp2tmajgaVa6wZAN2BuXvFJ0RVCOJSCKrqYryN+Qmt9Smudgvl+kWHZ2mjAPeNvD+B8XiuVMV0hhEO5l58BK6X6Af2yPLRAa70g4++KwLksy+KAp7Ot4gNgvVJqIFACyPPe84VedE/O71rYL2F1j4RMtXUIheLS2iG2DqHg2fcxlfvmoGkViHs5kJZRYBfk2fDOugNfaK2nKaWeBf6tlKqrtb7jVealpyuEcCgFePZCPOCbZd4n47GsegOtAbTWu5VSRYFHgMQ7rVTGdIUQDqUAx3T3AzWUUlWUUm6YD5RFZmtzFmiR8bq1gKLAxbutVHq6QgiHUlA9Xa11mlLqDSAWcAaWaK1/VUqNBQ5orSOBwcBCpdTbmA+q/UvnceNJKbpCCMdSgAPeWutoIDrbY2Oy/P0b0PRe1ilFVwjhUORnwEIIYUX2/jNgKbpCCMdi3zVXiq4QwrFIT1cIIaxIiq4QQliRFF0hhLAiuQW7EEJYkfR0hRDCiqToCiGEFdl5zZWiK4RwLNLTFUIIK3KSA2lCCGE9dt7Rtd/r6e7etYMu4W3pFBrMV0sW5liekpLCqOHv0Ck0mFd6duX8ectrCydcOE/Acw35+qslmY99+58v6d6xPT06hfLeiCEkJycXeh53E9ToUX5Z3JvDn/dhSNcmOZZX8nInenIX9n32L2I/7krFR0pmLls9vhMXVgxk+dgO1gw5V7t27iC8fWtC27ZiyaKcF+FPSUlh+JC3CW3bip49unA+Pg6ApKTL9H2lF881eYpJ48daPCc1NYVxH7xHWEgwEe3bsHFDrHVy2bGd0HbBhLQOYvHC3HMZOvgtQloH8UK3zsRn5AKweOF8QloHEdoumF07d2Q+fvXqVQa/9SZhIa0Jb9+GX37+CYBjR4/Ss0dXOoa3Z+BrA7h+/XrhJwj8sGsHHULbEB4SzBeLc39vjRz6NuEhwbz0QlfOx5vfW4cPHaRHlwh6dImge+dwtmzaYPE8k8lEjy4deOuNAVbJ406cnFS+J5vEZ5NXzYPJZGLqpI+YMWc+3y5fw/qYaP44ecKiTeSq5biXcuf7yFi6v/ASn86aZrF81rQpPNu0WeZ8YqKRpd/+h8+/XsY330eSnm5iQ6zFFdusyslJMfONIMJGfU+Dvkvo7F+LmpXKWrSZ2M+frzf+SpMBXzDh692MfaV55rIZy/bRe4rt4v+HyWRi0vixzJm7kOWr1xKzLoqT2bbVqhXfU8rdncjo9bzQ8yVmzTBvqyJuRXjtjUG8PWRYjvUuWvAZZcqUZfXaWJavjqJho5wfSoWRy4TxY5n72SJWRkYRE72Wkycsc1m5fBnu7u6sjdnAi73+xczp5ls3nTxxgpjoKFZERjF3/iImfPQhJpMJgCkTx9P0+WasXhvDsuWrqVK1GgAfjhnFoLcHs3zVGgJbtuSLJYuskuPkCeOYPXcBy1auITYmilPZttfqld9Tyt2DVWtj6fFiLz6Zac6xevUafPXNMr5ZupJP5i5gwrgPSEtLy3zet1//mypVqxZ6DnlRKv+TLdhl0f3t8CF8fCtR0ccXV1c3goLbsH3rZos2O7Zupm37cAACWrbiwL49/HPt4G1bNlKhYkWqVKtu8RyTyURy8t+kpaXx999/U66cl1XyyU3jx8tz8vxlTidcITUtnWXbjhLynGW8NSuVZdvPZwHY9vNZQp69vXzrz2e5djPFqjHn5vChg/hWqoSPr3lbBbdpy9YtmyzabN2yifah4QC0DApm397daK0pVrw4DZ5qSBE3txzrXb1yBa/0Md8v0MnJidKlS1snF9/K5lzc3Gjdtl2OXLZs3kxoWAQAQa2C2bfHnMvWLZto3bYdbm5u+Pj44utbmcOHDnLt2jX+97/9RHTsBICrmxvu7uabx545c5qGjRoD8OyzTdm0YX2h5/jr4YP4+lbCJ+O91ap1W7Zle29t27KZkFDzTW9bBAWzL+O9VbRYMVxczCOSyckpFgesjMYEdu3YRnhEp0LPIS8FeOeIQmGXRfdiohEvg3fmvJfBm4sXE3O0MXib27i4uFCyZCmuJCVx8+YN/v35Ynr3f82ivZeXgRd6vUx4mxaEBPlRomRJnn72nq49XKAqPFKSuIvXMufjL16jYtmSFm0OnUokrGkNAMKa1sC9RBHKlCpq1TjzkphoxOBdPnPeYPDmotGYrU0i3hlt/tlWSUlJd1zntatXAfh0ziy6d+nA0HcGcenPPws++GwSjUa8y2fd7wwYc+RitMylVCmSki5jNN7eHwEM3gYSjUbi4+IoXboMY0aNpEvHcD4YM4qbN28CUK16DbZsNhf19bExJCRcKOwUSUxMtIjTy8scp2Wb29s063sL4PDBX+gSEUK3TmGMHP1+ZhGeNmUib749BGUH17J12J6uUurlggykoCz67FO6vdiL4sVLWDx+9eoVtm/dzIq1G1i7fit/37rFuqjstzuyLyMXbKVZfV92z+1Fs/q+xF+8hin9rncCcQhpJhNGYwJPPNmAb5euoP4TTzJj2hRbh3VfTKY0jh75jc7durN0+SqKFSuWOe794bjx/Pe7b+jWuQM3b97A1TVnj9/e1K3/BEtXruWrb5by+eKFJCcns2PbFsqUKUOt2nVsHR5g/maU38kWHuTshQ+Bz3NbkPVe8tM/mce/Xul7Tysu52Ug0ZiQOZ9oTMgxFFDOy4AxIQEvgzdpaWlcv34ND09Pfj18kM0b1zNn5jSuX7uGk5PCza0IZcqWpUKFipQuUwYA/8AgDv3yM23ahd5TbAXl/J/X8SlXKnO+YrlSxF+yPJBy4a8bdBu7GoASRV0Jf/4xrtyw7cG/7Ly8DBiz9NCMxgTKGQzZ2niRkHABg/ftbeXp6XnHdXp6elK0WDFatGwFQFBwa1atXF4o8VvEaTCQcCHrfmfEkCMXg2Uu167h6Vkag8G8P/7DmGDEy2DAYPDGYPCmfv0nzLm0ap1ZdKtUrcb8heYDvadP/8H2bVsLOUPztsgaZ2KiOU7LNuZtasj23sqqStVqFC9enJMnjvPLzz+xfesWdu3cTkpyCtdvXOe9kcMYN9E2H5QP9dkLSqmDd5gOAYY7PU9rvUBr3Uhr3eheCy5ArTp1OXf2DOfj40hNTWFD7Dqa+QdYtGnmF0D0mlUAbNm4nkaNn0Ypxfwl/2FV9EZWRW+k6ws9eal3Pzp3ewGDd3kOH/qFv2/dQmvNgX17eLSK7Qb9Dxy7QPWKpans7YGrixOd/WoStdvygEZZ92KZO9DQbk/zZewhG0R6d3Xq1uPsmTPEx5m3Vey6aPz9Ay3a+PkHsiZyFQAbN8TSuMkzdx1PU0rR3C+AA/v3AbBvz26qZhx8Kkx16tbj7NnTxMWdIzUlhZjoKPwCLHPxDwgkcvVKADasj6XJ0+Zc/AICiYmOIiUlhbi4c5w9e5q69erzSLlyGLy9Of3HKQD27tlN1WrmXC5dugRAeno6C+fPo3PXboWeY+069Th39vb2Wh8TTXM/y/dWc/8A1kaaP+w3Zdle8XFxmQfOLpyP5/TpU1SoUJE3Br1D9IatrFm3ifGTp9G48dM2K7hg/2O6efV0DUAwcDnb4wr4oVAiwjyONGT4KAa91pf09HRCwiKoWq0GC+Z+Qs3adWjuH0j78I58OHo4nUKDcXf3ZNykqXddZ916TxDYshUv9eiEs7Mzj9WsRXjHLoWVQp5M6Zq352xkzYROODs58WXsIY6cucR7vZry4+8JRO05SfMnfBn7SnO01uw8FMdbczZmPn/jtO485luGksVcOfH1AAZMj2Hj/05bPQ8XFxeGv/serw3oTbopnbCIjlSrXoO5c2ZTu05d/AMCCe/QidEjhxHathXuHh5MmjI98/ltgwO5cf0GqampbNm8ibkLFlOtWnUGvT2Y0SOHM3XyBEqXKcMH4yZYJZeRo8bwar8+pKebCI/oSPXqNfj0k1nUqVMX/8AWRHTsxKgRQwlpHYS7hwdTps4AzEf2W7VuQ0RoW5ydnXl39BicnZ0BGPHue4wcPoTU1FR8fHwZ+9FEAGKi1/Ldt98A0KJlEOERHa2S49CRoxn4ah9M6emEhnegWvUafPbpbGrVqYuffyBhEZ0YM2o44SHBuLt7MGGK+WyTn3/6H18uWYiLqytKKUa8OwZPKxzgvFf23tNVd7tbsFJqMfC51npnLsu+0Vr3yOsFLt80OdwgZIXw6Xk3eghdWjvE1iEUOHv/ddL9Sk1Lt3UIhaJU0QffYA3Hbcl3zfnfewFW30Hu2tPVWve+y7I8C64QQlibvfd05WfAQgiHYu/fbqToCiEcilxlTAghrMjOa64UXSGEY5GerhBCWJGd11wpukIIxyIH0oQQwopkeEEIIaxIiq4QQliRnddcKbpCCMciPV0hhLAiO6+5UnSFEI7F3s9esP29NYQQogA5KZXvKS9KqdZKqWNKqRNKqRF3aNNFKfWbUupXpdQ3ea1TerpCCIdSUMMLSiln4FMgCIgD9iulIrXWv2VpUwMYCTTVWl9WSuV5t1vp6QohHEoB3jmiCXBCa31Ka50CfAeEZWvTF/hUa30ZQGudSB6k6AohHIqTyv+klOqnlDqQZeqXZVUVgXNZ5uMyHsvqMeAxpdQupdQepVTrvOIr9OGFIq6OV9f/ihpq6xAKRZnw2bYOocBdWjXQ1iEUDvs+VmRT93IgTWu9AFjwAC/nAtQA/AEfYLtSqp7WOumO8T3AiwkhhN1R9/AvD/GAb5Z5n4zHsooDIrXWqVrrP4DfMRfhO5KiK4RwKPcyvJCH/UANpVQVpZQb0A2IzNZmFeZeLkqpRzAPN5y620rl7AUhhEMpqF+kaa3TlFJvALGAM7BEa/2rUmoscEBrHZmxrJVS6jfABAzVWl+623ql6AohHEpB/iJNax0NRGd7bEyWvzXwTsaUL1J0hRAOJT8/erAlKbpCCIdi7z8DlqIrhHAodt7RlaIrhHAsMrwghBBWZN8lV4quEMLByEXMhRDCiuz8OJoUXSGEY5GzF4QQwopkeEEIIazIzju6UnSFEI5FerpCCGFF9l1ypegKIRyMs52PL9jt9XR37dxBeEhrQtu0YsminBd2T0lJYfjgtwlt04qe3btwPj4OgKSky/R9uRfPNX6KSePHWjzn9f596NIhjI5hIXz04fuYTCar5HInu3ZuJywkmPZtgu6Y47DBb9G+TRAvdu9MfJYc+7zck2cbN2BithztQVDDyvwyvyeHF/ZiSOeGOZb7litJzMQO7J7dnX1zehDcqDIAri5OzH+rJfs/7cHeT7rTrF72O6NYl6Pugz/s3EGH9m0IbxfMF4sX5liekpLCyKFvE94umJd6dOV8vPm63Xt27+LFrh3p2iGUF7t2ZP/ePQDcuHGDHp0jMqcWzZ9l2uQJVs0pqwK8R1qhsMuiazKZmPTRWObMW8jyyLXEREdx8uQJizarVnxPKXd3Itet54WeLzFr+jQAirgV4bWBg3h7yLAc6508bSZLV6zm+1VruHz5LzbExlgln9yYTCYmfjSWT+ctYkVkFDHRa3PkuHLFMtzd3VmzbgMv9vwXs6ZPBcw5vj5wEO/kkqOtOTkpZr7qT9j7q2nw6n/o3PwxavqWsWgzvFsTlu84zrNvfkuvyTHMei0AgFeC6wLQ+PVvCBm9ikl9mtnsd/SOug+aTCYmTxjH7HkLWLZqDbHrojiVLa/VK76nlLsHq6Ji6dGzF5/MNO93np6lmfHJPP67IpIPPprImFHDAShRogTfLFuZOZUvX4GAFkFWzSsrpfI/2UKeRVcpVVMp1UIpVTLb43negO1+HT50EN9KlfDx9cXV1Y3gNm3ZunmTRZutmzfRPiwcgJatgtm3dzdaa4oVL06DpxpSpIhbjvWWLGlOIS0tjbTUVJsOuJtzrJwlx3a55LiZ9mERQG45NsKtSBFbhH5XjR8zcPJ8EqcTrpKals6y7ccJeaaqRRutNe7FzdvHo4QbF/66AUDNSmXY+ou5t3jxyi2uXE+mYQ2DdRPI4Kj74K+HM/LyMefVqnVbtm3ZbNFm29bNhISab3rbIiiYfXv3oLWmZq3alPMy32G8WvUaJP+dTEpKisVzz5z+g8t//UWDho2sk1AunJTK92ST+O62UCn1JrAaGAgcVkplvf1woX1/SEw0YvAunzlvMHhzMdGYrU0i3hltXFxcKFmyFElJSXmu+7V+vWnh15TiJUrQslVwgcZ9LxITjXh7e2fOGwwGEnPkaMwlx8tWjfNeVShbkrg/r2fOx/95nYplS1i0Gf/1XroFPM6JL19h5YehvPPZVgAO/XGRkGeq4OykqGxwp0F1L3wesfistxpH3QcTjYkYDLf3O6/c9jujEYPBMq8r2fLatGE9NWvVws3N8oNlfUw0QcFtbNqhedh7un2BhlrrcMz3AXpPKTUoY9kdQ856W+PcxsJsae6CxWzYsoOUlJTMMSlhXV38Huc/G49Q/aUlRLwfyeLBwSgFX67/jfg/r7NrVjc+7tecPUcuYErXtg63wD3s++DJE8f5ZOY03h3zYY5l62PWEdy2nQ2iuu1hH9N10lpfB9Ban8ZceNsopaZzl6KrtV6gtW6ktW70Sp9+d2p2R15eBowJFzLnjcYEynkZsrXxIiGjTVpaGtevX8PT0zNf6y9SpAj+AS3YumVT3o0LiZeXgYSEhMx5o9GIV44cDbnkWNqqcd6r85euW/ROKz5SkvhLNyzavNSqNst3HAdg79EEiro584h7MUzpmmELd/DMwG/pMm4tniWLcDw+yZrhZ3LUfdDL4IXReHu/S8xtvzMYMBot8/LIyMuYkMDQtwfy4fhJ+PhWsnje78eOYjKlUat2ncJNIg/OSuV7soW8iq5RKfXkPzMZBTgEeASoV1hB1albj7NnzxAfF0dqagqx66LxDwi0aOMXEMia1asA2Lg+lsZPP3PXT66bN29w8WIiYN6Rdm7fxqNVqt6xfWEz53ia+LhzGTlG4ZdrjiuB/OVoDw78bqR6RU8qG9xxdXGic/MaRO21vDnquYvX8H/SfGfrx31LU9TVmYtXblGsiAvFi5jPYgx80pc0UzpHz/1l9RzAcffB2nXqce7M7bzWx0TT3D/Aok1z/wDWRq4GYNOGWBo3Med17epV3npjAG8MeocnGzyVY92x66IIbm3bXi4U6N2AC4Uy31ftDguV8gHStNYJuSxrqrXeldcL3Ey9ywvcxY7t25g6eQLppnTCIjrSp/8A5s6ZTe06dfEPCCQ5OZnRI4dx7MgR3D08mPTxdHx8zW/ktq0CuXH9BqmpqZRyL8XcBYvx9PDkzdcHkJqSQrrWNGrShCHDRuLicu+nKqsCOv16x/ZtfDx5AukmE2ERHenb/1XmzpmVkWMLkpOTGTVyaGaOkz+ekZljm1aB3Lh+PTPHeQuWUK1a9QeKp0z47IJIi+BGlfm4X3OcnZz4csOvTPnvAd578Wl+PJ5I1N4/qOlbhrlvBlKiqCsaGLVkF5t+Okslr1KsGRdOutacv3SdV2du4uzFaw8Uy6VVA+/7ufa8Dz7IsMvOHduYPmUiJlM6oeEd6N1vAJ99Optatevil5HXmHeHc+yoOa8JU6bh4+PLogXz+GLRQipVrpy5rjmfLaJM2bIAhLUJYtbc+Q/0QVKqyIOXwncij+b7P2d6aE2rl967Ft2CcL9F154VVNG1NwVVdO3JgxRde+aIY91QMEV38Jpj+f7Pmdb+cau/meUXaUIIh2LnP0iToiuEcCx2fthDiq4QwrG42HnVlaIrhHAodl5zpegKIRyL3IJdCCGsyM5rrhRdIYRjkbMXhBDCiuz9IuZSdIUQDsXOa64UXSGEY7H3X4xK0RVCOBTp6QohhBVJ0RVCCCuy98uf2uWNKYUQ4n45O+V/yotSqrVS6phS6oRSasRd2nVUSmmlVJ43h5OerhDCoRTUL9KUUs7Ap0AQEAfsV0pFaq1/y9auFDAI2Juv+AokOiGEsBMFeOeIJsAJrfUprXUK8B0Qlku7ccBk4O/8xFfoPV17P33jfmgc8wLSicvfsHUIBa5sE8e8iPmlvZ/YOgS7dS8dXaVUPyDrjRwXaK3/uZtuReBclmVxwNPZnv8U4Ku1jlJKDc3Pa8rwghDCoTjdQ0cvo8De1y3LlVJOwHTgX/fyPCm6QgiHUoAnL8QDvlnmfTIe+0cpoC6wNeOMCW8gUikVqrU+cKeVStEVQjgUl4I7UXc/UEMpVQVzse0G9Phnodb6CuY7owOglNoKDLlbwQU5kCaEcDBK5X+6G611GvAGEAscAZZqrX9VSo1VSoXeb3zS0xVCOJSCvIi51joaiM722Jg7tPXPzzql6AohHIqd/yBNiq4QwrHY+5ipFF0hhEORe6QJIYQVSdEVQggrsu+SK0VXCOFg7LyjK0VXCOFY7P16ulJ0hRAORc5eEEIIK5IDaUIIYUUyvCCEEFYkwwtCCGFF9t7TtfcPhUy7dm4nLCSY9m2CWLIo5zWHU1JSGDb4Ldq3CeLF7p2Jj48DICnpMn1e7smzjRswcfxYa4edw66dOwgPaU1om1Z3zGP44LcJbdOKnt27cD5LHn1f7sVzjZ9iUpY8bt26xcBX+xPRvg0dw0KYNWOa1XK5kx927aBDaBvCQ4L5YvHCHMtTUlIYOfRtwkOCeemFrpyPN1+i9PChg/ToEkGPLhF07xzOlk0brB36HQU9V4tfVr7H4dXvM+TloBzLK5UvTfRnA9n335HELhxERS/PzGUfvRnGgWXvcmDZu3Rq9ZQVo87drp07CG/fmtC2d9kHh7xNaNtW9OyRbR98pRfPNbHcBwFSU1MY98F7hIUEE9G+DRs3xFoll9yoe5hs4aEouiaTiYkfjeXTeYtYERlFTPRaTp48YdFm5YpluLu7s2bdBl7s+S9mTZ8KQBG3Irw+cBDvDBlmi9AtmEwmJn00ljnzFrI8ci0x0VE58li14ntKubsTuW49L/R8iVnTzUW0iFsRXhs4iLdzyaPXyy+zcs06vvt+Bb/89CM7d2y3Sj65MZlMTJ4wjtlzF7Bs5RpiY6I4lS3H1Su/p5S7B6vWxtLjxV58MtO8rapXr8FX3yzjm6Ur+WTuAiaM+4C0tDRbpGHByUkxc0QXwt6YS4OOH9G5dUNqVvW2aDPx7Qi+jtpHk64TmbBgHWMHmq/81/r5OjxZy5enu02iec+pvNWrBaVKFLVFGkDGPjh+LHPmLmT56rXErLvLPhidsQ/OyLIPvpH7PrhowWeUKVOW1WtjWb46ioaNmlgln9w4K5XvyRYeiqJ7+NBBfCtVxsfXF1dXN4LbtGPr5k0WbbZu3kz7sAgAWrYKZt/e3WitKVa8OA2eaoRbkSK2CN2COY9KWfJom0sem2gfFg7klkdDihRxs2hfrFgxGjd5BgBXVzdq1qpNojHBKvnk5tfDB/H1rYSPjznHVq3bsm3rZos227ZsJiTUfH+/FkHB7Nu3B601RYsVw8XFPOKVnJxiN18TG9d9lJPn/uR0/CVS00wsi/2REP/6Fm1qVi3Ptn3HANi2/3dC/OsBUKuqNzt/PIHJlM7Nv1M4dDyeVs/VsnoO/8h1H9ySbR/cson2oeEAtAzKZR90c8ux3tUrV/BKH/OtxpycnChdunSh53InBXU93cKSZ9FVSjVRSjXO+Lu2UuodpVTbwg/ttsREI97et3sWBoOBxERjLm3KA+Di4kLJkqVISrpszTDzlJhoxJARI4DB4M3FHHkk5pJHUr7Wf+3qVbZv20KTp58tsJjvVWJiIoYs28rLy0CiMee2MmTL8UpGjocP/kKXiBC6dQpj5Oj3M4uwLVXw8iDOeHtfijdepmI5D4s2h36PJyzwSQDCAp/AvWQxyniU4ODv5iJbrKgrZT1L4NfoMXy8bVeQct0Hc2yfe9sHr129CsCnc2bRvUsHhr4ziEt//lnwweeTuod/tnDXoquUeh+YDcxTSk0E5gAlgBFKqVFWiE/kU1paGiOGDab7Cz3x8fXN+wl2qm79J1i6ci1ffbOUzxcvJDk52dYh5cvIGStp1rA6u78dTrOG1Yk3XsZkSmfTnqPE7PyNLV8M5suJL7P34B+YTOm2DrdApZlMGI0JPPFkA75duoL6TzzJjGlTbBbPw97T7QQ0BZoDrwPhWutxQDDQ9U5PUkr1U0odUEodWJzLQP298vIykJBw+yuz0WjEy8uQS5sLgLkAXb9+DU9P2/UocuPlZcCYESOA0ZhAuRx5eOWSh2ee6/7ogzFUqlSZF3q+VKAx3ysvLy+MWbZVYqIRL0PObWXMlqNHthyrVK1G8eLFOXnieKHHnJfziVfwMdzelyoaShN/8YpFmwsXr9BtyCKe7T6Z9+esAeDK9VsATFkcyzPdJhHy6hyUUhw/m2i94LPJdR/MsX3ubR/09PSkaLFitGjZCoCg4NYcOfJbwQefT06ofE+2ie/u0rTWJq31TeCk1voqgNb6FnDHj2ut9QKtdSOtdaPeffrdqVm+1albj7NnTxMfd47U1BRi10XhFxBo0cYvIJA1q1cCsHF9LI2ffsZuxgT/Yc7jDPFxcRl5ROOfax6rgPzn8ensmVy7fo2hI94trNDzrXadepzLkuP6mGia+wVYtGnuH8DayNUAbNoQS+Mm5hzj4+IyD5xdOB/P6dOnqFChotVzyO7Ar2eoXqkclSuUxdXFmc7BTxG19aBFm7KeJTK309BXgvly9R7AfBCujEcJAOrWqEDdGhXYuPuodRPIok7depw9k20f9M+2D/oHsiZyFQAbs2yfO1FK0dwvgAP79wGwb89uqlatVmg55MXee7pKa33nhUrtBQK01jeVUk5a6/SMxz2ALVrrPM9/uZXKnV/gHuzYvo2PJ08g3WQiLKIjffu/ytw5s6hdpy7+AS1ITk5m1MihHDtyBHcPDyZ/PCPza3abVoHcuH6d1NRUSrmXYt6CJVSrVv2+Y9EPkNKO7duYOnkC6aZ0wiI60qf/AObOmZ2RRyDJycmMHjksM49JH0/PzKNtq0BuXL+RmcfcBYspWaIkrVv6U6VKVVwzDnB07f4CHTp1vufYTKYC2VTs3LGN6VMmYkpPJzS8A737DuCzT2dTq05d/PzNOY4ZNZxjR4/g7u7BhCnT8PHxJWrNar5cshAXV1eUUvTt/xr+gS0fKBavZ98skJyCn6/Nx0M64eyk+HL1HqYsjuW9V9vx429nidp2iIiWTzJ2YChaw84fT/DWxKWkpKZRxM2F3d8OB+Da9b8ZOP47Dv4en8er5e3S3k/u+7k7tm9j6pQs+2C/O+yDRzP2wSlZ9sHgLPtgKfM+WK1adc6fj2f0yOFcv3aV0mXK8MG4CZQvX+GeYyvu9uClcMORP/O9IwfVesTqpTevoltEa51jUE0p9QhQXmt9KK8XKKiia08epOjas4IquvakoIquvXmQomvPCqLobjqa/6Lboqb1i+5dDw3nVnAzHv8TsN3hSSGEuANbnZWQX7Y/H0cIIQqQnR3KyUGKrhDCoUhPVwghrMjJvmuuFF0hhGORi5gLIYQV2XfJlaIrhHAw0tMVQggrsu+SK0VXCOFo7LzqStEVQjgUGV4QQggrsu+SK0VXCOFo7LzqStEVQjgU+UWaEEJYkZ0P6T4cN6YUQoj8KshbsCulWiuljimlTiilRuSy/B2l1G9KqYNKqU1Kqcp5rVOKrhDCoSil8j3lsR5n4FOgDVAb6K6Uqp2t2U9AI611feB7IM+bw0nRFUI4lAK8XU8T4ITW+pTWOgX4DgjL2kBrvSXjdmYAewCfvFZa6GO615PTCvslrK64m7OtQygUTvZ+eab78Ne+ObYOoVCU6fzgN3y1R7dWPfg9Fe9lL1ZK9QOyvugCrfU//7kVgXNZlsUBT99ldb2BdXm9phxIE0I4lnuouhkF9oE/wZRSLwKNAL+82krRFUI4lAI8ZSwe8M0y75PxmOXrKdUSGAX43ekWZ1nJmK4QwqEU4JjufqCGUqqKUsoN6AZEWr6WagDMB0K11on5iU96ukIIh1JQ5+lqrdOUUm8AsYAzsERr/atSaixwQGsdCXwMlASWZZwNcVZrHXq39UrRFUI4lIL8RZrWOhqIzvbYmCx/t7zXdUrRFUI4FHv/RZoUXSGEQ7HzmitFVwjhYOy86krRFUI4FLmIuRBCWJF9l1wpukIIR2PnVVeKrhDCochFzIUQworsfEhXiq4QwrHYec2VoiuEcCx5XZzc1qToCiEcip3XXCm6QgjHYuc1134v7bjnhx1079COrmGt+ffnC3MsT0lJYcyIwXQNa03fXt24cP72ZS5PHD9G/3/14MXOofTqEk5ysvkSl6mpKUz+6H26RbSlR4cQtm5ab7V8/rFr5w4i2rcmtG0rPl+U89rJKSkpDB/yNqFtW9GrRxfOx8cBsOeHXfTo0oEuEe3p0aUD+/buyXzOnNkzaNPSn6ZNnrJaHlndb05JSZfp90ovmjZ5iknjx1o8JzYmmi4dQukUHsKs6VOtkgfArp3bCQsJpn2bIJbcIZdhg9+ifZsgXuzemfiMXAAWL5xP+zZBhIUE88OuHZmPf/3vL+kYHkKHsHb8599fWKzv26//TXj71nQIa8eMaXneXqvABTXw4ZdPu3B4XleGdHgix3LfR0oQMy6E3dM7sG9mR4Ibmi8v2615dfbM6JA53VjRl/pVylo7/NwV5J0pC4Fd9nRNJhPTJ41nxtyFeBkM9OnZlef9AqhStXpmm7WrllPK3Z3/ro5hY2w082ZPZ+ykaaSlpTFu9AhGj5tIjcdqciUpCRcXc5pfLV5A6TJl+G5lNOnp6Vy9csXqeU0eP5a5C5Zg8DbwYrfO+AUEUrXa7bxWrfged3d3IqPXE7suilkzpjF56gw8S5dm1px5lPMycOL477w+oA+xm7YD0NwvgK7dXyC8XWur5vOgORVxK8Krbwzi5InjnDj+e2b7pKTLzJr2MV//dzmly5RhzKjh7N2zm6efebbQc5n40Vg+W/g5Bm8DL3TthF9AINWy5LJyxTLc3d1Zs24DMdFRzJo+lSnTZnLy5Ali10WxfHUUFxON9O/zMqujYvnj1ElWLF/Gf75dhqurK68P6ENzvwAqVarM/n172LplE0uXR+Lm5sZfly4Van7ZOTkpZvZ/nnbvRxF/6QY7P45g7b4zHI1LymwzvMtTLN91koUxR6jp48mqMW2o2e9bvtt+gu+2nwCgTuXSLB0ZzME/rBv/ndj7KWP33NNVSn1VGIFkdeTXQ/j4+lLRxxdXVzdatmrLzq1bLNrs3LaZNiHme8T5t2jF//btQWvN/j0/UK3GY9R4rCYAHp6eODub72kWFbmSni/3BcDJyQnP0qULOxULhw8dxKdSJXx8zXkFt2nL1i2bLNps3bKJkNBwAFoEBbN/72601tSsVZtyXgYAqlWvQfLfyaSkpABQ/4knKVfOy6q5/ONBcipWvDgNnmqIm5ubRfv4uDh8K1emdJkyADR55jk2byz8byWHDx3Et1LlLLm0Y+vmbLls3kz7sAgAWrYKZl9GLls3byK4TTvc3Nyo6OOLb6XKHD50kFOnTlKvXn2KFSuGi4sLDRs1ZlNGLkv/+y0v9+6XmX+ZstbtKTauUY6TF65w2niN1LR0lu08ScjTj1q00Rrci5nj8yjhxoW/buRYT5dm1Vm246Q1Qs6XAryIeaG4a9FVSkVmm9YAHf6ZL6ygLiYa8TKUz5wvZzBw8aLRss3FRLwM3gC4uLhQomQpriQlce7saZRSvPN6X17p0Ymvv1wMwLVrVwFYNO8TXunRidHD3uavS38WVgq5uphoxNv7dl5eBm8SjdnySkzMbOPi4kLJkqVISkqyaLNpQyw1a9XOUaxsoaByysrXtxJn/viD8/FxpKWlsXXzRhISLhRK/FklJhrx9vbOnDcYDCQmGnNpkz2Xy3d8bvXqj/Hjj/8jKekyt27dYueO7RgTEgA4c/o0P/7vAC9270zvf73I4UMHCz3HrCqUKUHcn7eLaPylG1QsU8KizfjvDtDNvwYnFvVg5XtteGfhDznW0+n5aizdcaLQ480vJ5X/ySbx5bHcB7gKTAemZUzXsvydK6VUP6XUAaXUga+W5ByPLUxpaSYO/vwjYz6awtzF/2b7lk0c2LcHU5qJRGMCdes/yZJvvqdu/Sf4dKb1xgoLyskTx5k9Yxqj3v/Q1qEUGncPD0a+9z4jhr5D75deoEKFijg7PZx3YK5arRovv9KHV/v15vUBfXj88Zo4OZnfdiaTiatXr/Dvb5by1uBhDBvyFlprG0dsqUuz6vxn8zGq9/mGiHHrWPxWgEUPsXGNctxMTuO3s5dtF2QO9j2om1fRbQT8D/NN165orbcCt7TW27TW2+70JK31Aq11I611o16v9L3noMp5GUg03u7ZXDQaKVfOYNmmnBeJRnOPIS0tjRvXr+Hh6YmXwcATDRriWbo0RYsV49mmzfj96G94eHpStGgx/AKDAAhoGcyxo7/dc2wPopyXwaLHlmhMwMuQLS8vr8w2aWlpXL9+DU9PTwCMCQkMfusNxk6YjK9vJavFfTcPmtOd+PkH8tU3S/ny6/9S+dEqVHr00YIOPQcvLwMJGb1QAKPRiJeXIZc22XMpfdfnRnTszLdLV7Dky68p5e5B5YxcDAYDLVoGoZSiXr36OCknLl+2XvE6/9cNfB653bOtWLYE8dmGD15q+TjLd50CYO+xRIq6OvOIe9HM5Z2bVberXi485MMLWut0rfUM4GVglFJqDlY4+Fazdl3OnTvL+fg4UlNT2Lg+mqZ+ARZtmvoFsG7tagC2blrPU42fRilFk2ebcurEcf6+dYu0tDR++vEAj1aphlKKps39+enAPgD+t28Pj1apVtipWKhTtx7nzpwhPs6cV+y6aPz8Ay3a+PkHsjZyFWAeRmjc5BmUUly7epU3X+/PwLcG82QD25ylkJsHyelu/jmodPXKFZb991siOnQqlPizqlO3HmfPniY+7lxGLlH4BWTLJSCQNatXArBxfSyNnzbn4hcQSOy6KFJSUoiPO8fZs6epW6++RS4XLpxn86b1tGnbHoCAwJbs37cXgDOn/yA1NZXSVjzOcOD4RaqX96CyVylcXZzo/Hw1ovadsWhz7uJ1/OtXBOBxH0+Kujlz8crfgLlodWxa1a7Gc8He+7mg7uXrjFKqHdBUa/1ufp9z8XrafX1f2r1zO7OmTSLdlE67sAhe6t2fRfM+oWbtOjzvF0hycjLj3hvB8WNHcPfw4IMJU6noYz6dJTZ6Df/+fCFKKZ5t2ozXBg0BIOHCeca9N4Lr167hWbo0I9//CO/yFe45tuJu9/9Vd+f2bUydMoF0UzqhER3p028A8+bMpnaduvgFmPN6b+Qwjh49goeHBxOnTMfH15dF8+exZPECKlWqnLmuufMXU6ZsWWZO/5iYqLVcvJhIuXJehHfsxIDXBt53jNbKCaBdcCA3rt8gNTWVUqVKMXfBYqpWq87IYe/w+7FjAPQb8BrBbdrdV2z3em3VHdu38fHkCaSbTIRFdKRv/1eZO2cWtevUxT+gBcnJyYwaOZRjR8z73eSPZ2TmsnD+PFavXI6zizNDh7/L8838AHi5V4/Ms2gGDxuZeRZGamoK749+l2PHjuLq6so7Q4bR5On8naFRpnPO09nuR3BDXz5+5VmcnZ34cuMxpnz/E+91b8iPJ/4kav8Zavp4Mvf15pQo6opGM+rLvWz62Xx6ZrO65fmoZxP8hq8ukFgAbq3q98C18MKVlHzXnPIeblavvfdUdO/H/RZde/YgRVdYl71f0Pp+FVTRtTcFUXQTrqbmu+Z4u7tafQexy/N0hRDiftn7x6wUXSGEQ7H3LzdSdIUQDsXef5EmRVcI4Vjsu+ZK0RVCOBY7r7lSdIUQjsXez1iRoiuEcCh2XnPt93q6QgjhiKSnK4RwKPbe05WiK4RwKHLKmBBCWJH0dIUQwoqk6AohhBXJ8IIQQliR9HSFEMKK7LzmStEVQjgYO6+6UnSFEA7F3n8GXOh3jrAmpVQ/rbXDXVLfEfNyxJzAMfNyxJxsydF+BtzP1gEUEkfMyxFzAsfMyxFzshlHK7pCCGHXpOgKIYQVOVrRddRxJ0fMyxFzAsfMyxFzshmHOpAmhBD2ztF6ukIIYdek6AohhBU5RNFVSrVWSh1TSp1QSo2wdTwFQSm1RCmVqJQ6bOtYCpJSylcptUUp9ZtS6lel1CBbx/SglFJFlVL7lFK/ZOT0oa1jKkhKKWel1E9KqbW2jsURPPRFVynlDHwKtAFqA92VUrVtG1WB+AJobesgCkEaMFhrXRt4BnjdAbZXMhCotX4CeBJorZR6xrYhFahBwBFbB+EoHvqiCzQBTmitT2mtU4DvgDAbx/TAtNbbgb9sHUdB01pf0Fr/mPH3Ncxv5oq2jerBaLPrGbOuGZNDHKFWSvkA7YBFto7FUThC0a0InMsyH8dD/ib+/0Ip9SjQANhr41AeWMZX8J+BRGCD1vqhzynDTGAYkG7jOByGIxRd8RBSSpUElgNvaa2v2jqeB6W1NmmtnwR8gCZKqbo2DumBKaVCgESt9f9sHYsjcYSiGw/4Zpn3yXhM2CmllCvmgvu11nqFreMpSFrrJGALjjEe3xQIVUqdxjxsF6iU+o9tQ3r4OULR3Q/UUEpVUUq5Ad2ASBvHJO5AKaWAxcARrfV0W8dTEJRS5ZRSnhl/FwOCgKM2DaoAaK1Haq19tNaPYn5fbdZav2jjsB56D33R1VqnAW8AsZgPyizVWv9q26genFLqW2A38LhSKk4p1dvWMRWQpkBPzL2mnzOmtrYO6gGVB7YopQ5i7gRs0FrL6VUiV/IzYCGEsKKHvqcrhBAPEym6QghhRVJ0hRDCiqToCiGEFUnRFUIIK5KiK4QQViRFVwghrOj/AFGaFWzXXmuSAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "sns.heatmap(cm, annot=True, cmap=\"Blues\")\n", "plt.savefig(f\"images/{model_name}.png\")" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " precision recall f1-score support\n", "\n", " 0 0.86 0.82 0.84 1492\n", " 1 0.93 0.91 0.92 3476\n", " 2 0.92 0.89 0.90 3017\n", " 3 0.97 0.99 0.98 15462\n", " 4 0.88 0.87 0.88 2890\n", "\n", " accuracy 0.94 26337\n", " macro avg 0.91 0.89 0.90 26337\n", "weighted avg 0.94 0.94 0.94 26337\n", "\n" ] } ], "source": [ "print(classification_report(train[\"label\"], test[\"label\"]))" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.9429319968105707\n" ] } ], "source": [ "print(get_score(train[\"label\"], test[\"label\"]))" ] } ], "metadata": { "kernelspec": { "display_name": "pytorch", "language": "python", "name": "pytorch" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.8" } }, "nbformat": 4, "nbformat_minor": 4 }